montar-lista-com-acentos.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Montar lista de palavras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8183,\n",
       " 'a aba abacaxi abade abadia abafado abafar abaixar abaixo abajur',\n",
       " 'zombar zona zoólogo zoom zorro zulu zumbido zumbir zunir zunzum')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('fontes/com_acentos.txt', encoding='utf8') as fp:\n",
    "    com_acentos = [lin.strip() for lin in fp]\n",
    "\n",
    "len(com_acentos), ' '.join(com_acentos[:10]), ' '.join(com_acentos[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8144, 368)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "com_acentos = set(com_acentos)\n",
    "with open('fontes/2letras.txt', encoding='utf8') as fp:\n",
    "    bi_letras = [lin.strip() for lin in fp]\n",
    "bi_letras = set(bi_letras)\n",
    "sem_bi_letras = {p for p in com_acentos if len(p) != 2}\n",
    "filtradas = sem_bi_letras | bi_letras\n",
    "len(filtradas), len(filtradas)-7776"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Possíveis palavras de ortografia portuguesa ou inglesa:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "abjecto acta activar activo acto actor actriz actual actuar afectar afecto aspecto bráctea cacto colecta directo ducto dúctil erecto exacto facto factor factual factura fractal hectare impacte impacto insecto intacto invicto jacto lectivo lácteo néctar objecto olfacto pacto pactuar project reactor recto sector tactear tacto tecto tractor táctico táctil vector árctico "
     ]
    }
   ],
   "source": [
    "for p in sorted(filtradas):\n",
    "    if 'ct' in p:\n",
    "        print(p, end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "back boy brandy browser clown country cowboy crack derby ecstasy family faraday folk funk gay happy hay heavy hobby jockey k kaiser kart kelvin kent kg khmer king kit km/h know krill lady like link lobby look malawi market mayor money network nylon party player playoff punk quark rally ranking rock rocket say show skate ski smoking sputnik stock sweet swing system take very w wall was watt wave way web western whisky who with work www your "
     ]
    }
   ],
   "source": [
    "kwy = set('kwy')\n",
    "for p in sorted(filtradas):\n",
    "    if set(p) & set(kwy):\n",
    "        print(p, end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "abc abdómen about and art ateliê aux azt back bacon bad ballet band bang baú beat best bien big bip bit board bourbon box boy brandy buquê cabaret camp caput cartoon cash cep chic chip clip clown club clímax con coulomb country court cowboy crack crash cut cólon córtex dan deep derby design doping dossiê dumping déficit dólman ecstasy elétron eng expert family fan faraday fast fax flash folk front funk gang gay get gin gran grand group guichê habitat happy hard hay heart heavy hein hip hit hobby hoc holding hot iceberg inf inq ipê jet jihad job jockey just kart kelvin kent king kit km/h know lady land leasing led light link living lobby look lord lost lux látex líquen mach maison market match meeting mercê min mind mix modern money music must net network night nylon néon off party piraí pit playoff point pop porquê port pot premiê project pub public punk pág pólen quark quê raid rally ranking rap record regímen ring road robot rock rocket round rúmen san say sec sen set sex shop short show sig slogan smoking sob soft son split spot sprint sputnik srª staff stand station stock strip sub surf sweet swing sáb séc sémen tchan ten timing top tout trust turnê tórax und urv very vip você volt von watt way web western whisky with work www xerox zen écran éden íman índex "
     ]
    }
   ],
   "source": [
    "finais = set('aeiourláéózsmãô')\n",
    "for p in sorted(filtradas):\n",
    "    if len(p) <= 2:\n",
    "        continue\n",
    "    if p[-1] not in finais:\n",
    "        print(p, end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nacao'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import unicodedata \n",
    "\n",
    "def shave_marks(txt):\n",
    "    \"\"\"Remove all diacritic marks\"\"\"\n",
    "    norm_txt = unicodedata.normalize('NFD', txt)\n",
    "    shaved = ''.join(c for c in norm_txt if not unicodedata.combining(c))\n",
    "    return unicodedata.normalize('NFC', shaved)\n",
    "\n",
    "def ordem(txt):\n",
    "    return shave_marks(txt).lower()\n",
    "\n",
    "ordem('Nação')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7944, 168, ['a', 'aba', 'abacaxi'], ['zumbir', 'zunir', 'zunzum'])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('fontes/excluidas_a_mao.txt', 'rt', encoding='utf8') as fp:\n",
    "    excluidas = set(p.strip() for p in fp)\n",
    "    \n",
    "filtradas -= excluidas\n",
    "lt_fil = sorted(filtradas, key=ordem)\n",
    "len(lt_fil), len(lt_fil) - 6**5, lt_fil[:3], lt_fil[-3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7776, 0, ['a', 'aba', 'abacaxi'], ['zumbir', 'zunir', 'zunzum'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('fontes/excluidas_por_sorteio.txt', 'rt', encoding='utf8') as fp:\n",
    "    excluidas = {p.strip() for p in fp.read().split()}\n",
    "\n",
    "filtradas -= excluidas\n",
    "lt_fil = sorted(filtradas, key=ordem)\n",
    "len(lt_fil), len(lt_fil) - 6**5, lt_fil[:3], lt_fil[-3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7776, '11111', '66666')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import itertools\n",
    "dados5 = list(''.join(dados) for dados in itertools.product('123456', repeat=5))\n",
    "len(dados5), dados5[0], dados5[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nacao'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import unicodedata \n",
    "\n",
    "def shave_marks(txt):\n",
    "    \"\"\"Remove all diacritic marks\"\"\"\n",
    "    norm_txt = unicodedata.normalize('NFD', txt)\n",
    "    shaved = ''.join(c for c in norm_txt if not unicodedata.combining(c))\n",
    "    return unicodedata.normalize('NFC', shaved)\n",
    "\n",
    "def ordem(txt):\n",
    "    return shave_marks(txt).lower()\n",
    "\n",
    "ordem('Nação')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  7776  15552 100905 dadoware.txt\r\n"
     ]
    }
   ],
   "source": [
    "dadoware = list(zip(dados5, sorted(filtradas, key=ordem)))\n",
    "with open('dadoware.txt', 'wt', encoding='utf8') as fp:\n",
    "    for indice, palavra in dadoware:\n",
    "        linha = '{} {}\\n'.format(indice, palavra)\n",
    "        fp.write(linha)\n",
    "    \n",
    "!wc dadoware.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  7776  15552 100905 dadoware.txt\r\n"
     ]
    }
   ],
   "source": [
    "dadoware = list(zip(dados5, sorted(filtradas, key=ordem)))\n",
    "with open('dadoware_livreto.txt', 'wt', encoding='utf8') as fp:\n",
    "    for indice, palavra in dadoware:\n",
    "        linha = '#{}# {}\\n'.format(indice[2:], palavra)\n",
    "        fp.write(linha)\n",
    "    \n",
    "!wc dadoware.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('dadoware_topo_paginas.txt', 'wt', encoding='utf8') as fp:\n",
    "    for chave, palavra in dadoware[::216]:\n",
    "        fp.write('{} {}\\n'.format(chave, palavra))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}