nlptk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLPTK: тулкит для Natural Language Processing\n",
    "\n",
    "## Работа с классом Text: основные методы и свойства"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# минимально необходимые внешние модули\n",
    "#!pip install chardet\n",
    "#!pip install dawg\n",
    "#!pip install gensim\n",
    "#!pip install pattern\n",
    "#!pip install razdel\n",
    "#!pip install pymorphy2\n",
    "#!pip install cyhuspell\n",
    "#!pip install pyenchant\n",
    "\n",
    "import sys, os\n",
    "# сменим рабочую директорию на ту, где расположен каталог с библиотекой nlptk\n",
    "os.chdir(r'F:\\INSTALL\\Python3\\PROJECTS\\REPOS\\nlp_toolkit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#from nlptk.mining.utils import datapath\n",
    "\n",
    "#import nltk\n",
    "#nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# добавим этот путь в sys.path, чтобы интерпретатор увидел нашу библиотеку\n",
    "sys.path.append(r'F:\\INSTALL\\Python3\\PROJECTS\\REPOS\\nlp_toolkit')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "unable to import 'smart_open.gcs', disabling that module\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from pprint import pprint\n",
    "import nlptk\n",
    "from nlptk.mining.text import *\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentencizer': <function nlptk.misc.mixins.SentencizerMixin.sentencize_nltk(text, *args, lang='english', **kwargs)>,\n",
       " 'tokenizer': functools.partial(<function TokenizerMixin.toktok_tokenize at 0x0000000012FBA798>, strip='!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'),\n",
       " 'tagger': <function nlptk.misc.mixins.TaggerMixin.tagger_nltk(tokens, *args, lang='eng', **kwargs)>,\n",
       " 'lemmatizer': functools.partial(<function LemmatizerMixin.lemmatize_nltk at 0x0000000012FBA318>, pos=True, normalize_uppercase=<method 'capitalize' of 'str' objects>),\n",
       " 'speller': <nlptk.spelling.spellers.Speller at 0x15c8c688>}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# создаем экземпляр класса Prep c настройками по умолчанию\n",
    "prep = Prep()\n",
    "# это то, что будет использоваться при обработке текста\n",
    "prep.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# набор правил для очистки текста; применяются на этапе работы с необработанным текстом;\n",
    "rules_clean = OrderedDict(\n",
    "    contractions=True,        # замена английских сокращений общего характера (служебные слова) на их полную версию: didn't => did not\n",
    "    possessive_endings=True,  # удаление окончаний притяжательного падежа в анг.\n",
    "    #roman_numerals=(True,)    # удаление римских чисел\n",
    "    # здесь еще много всяких правил используемых по умолчанию\n",
    ")\n",
    "# набор правил для фильтра токенов; применяются на этапе запроса слов\\токенов\\лемм из тэгированного текста\n",
    "# могут быть включены на уровне составления вокабуляра класса Corpus \n",
    "# (по умолчанию фильтрация выключена, поэтому вокабуляр максимально полный)\n",
    "rules_filter = OrderedDict(\n",
    "        by_tagpos=(True,{},{\"FW\",\"POS\"}), # фильтр по тегам частей речи - игнорировать иностранные слова и possessive ending parent's\n",
    "        punctuation=True, # по умолчанию\n",
    "        short=(True,3),   # по умолчанию\n",
    "        stopwords=True,   # по умолчанию\n",
    "        ifnotin_lexicon=True\n",
    "        #trailing_chars=True\n",
    ")\n",
    "\n",
    "# классы которым передаются правила, сами же экземпляры классов передаются в конструктор класса Text\n",
    "clean = TextCleaner(rules_clean)\n",
    "filters = TokenFilter(rules_filter)   \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('hyphenation', (True,)),\n",
       "             ('accent', (True,)),\n",
       "             ('contractions', (True,)),\n",
       "             ('possessive_endings', (True,)),\n",
       "             ('tags', (True,)),\n",
       "             ('urls', (True,)),\n",
       "             ('numeric', (True,)),\n",
       "             ('roman_numerals', (False,)),\n",
       "             ('nonletter_sequences', (True,)),\n",
       "             ('quotes', (True,)),\n",
       "             ('multiple_whitespaces', (True,))])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean.rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "filters.rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "THE⁄The⁄DT PICTURE⁄Picture⁄NN OF⁄Of⁄IN DORIAN⁄Dorian⁄JJ GRAY⁄Gray⁄NNP BY⁄By⁄IN OSCAR⁄Oscar⁄NNP WILDE⁄Wilde⁄NNP\n",
      "The⁄The⁄DT it⁄it⁄PRP is⁄be⁄VBZ artist⁄artist⁄JJ is⁄be⁄VBZ the⁄the⁄DT creator⁄creator⁄NN of⁄of⁄IN beautiful⁄beautiful⁄JJ things⁄thing⁄NNS\n",
      "PREFACE⁄Preface⁄NN\n"
     ]
    }
   ],
   "source": [
    "# класс Text может принять в качестве ввода непосредственно текст (нужно указать это в параметре input='text', \n",
    "# по дефолту input=\"filename\")\n",
    "intake='''THE PICTURE OF DORIAN GRAY\n",
    "\n",
    "BY\n",
    "\n",
    "OSCAR WILDE. The it's artist is the creator of beautiful things. PREFACE'''\n",
    "text1 = Text(intake, prep, clean, filters, input='text', inplace=True)\n",
    "print(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('THE', 'DT', 'The'),\n",
       " ('PICTURE', 'NN', 'Picture'),\n",
       " ('OF', 'IN', 'Of'),\n",
       " ('DORIAN', 'JJ', 'Dorian'),\n",
       " ('GRAY', 'NNP', 'Gray'),\n",
       " ('BY', 'IN', 'By'),\n",
       " ('OSCAR', 'NNP', 'Oscar'),\n",
       " ('WILDE', 'NNP', 'Wilde'),\n",
       " ('The', 'DT', 'The'),\n",
       " ('it', 'PRP', 'it'),\n",
       " ('is', 'VBZ', 'be'),\n",
       " ('artist', 'JJ', 'artist'),\n",
       " ('is', 'VBZ', 'be'),\n",
       " ('the', 'DT', 'the'),\n",
       " ('creator', 'NN', 'creator'),\n",
       " ('of', 'IN', 'of'),\n",
       " ('beautiful', 'JJ', 'beautiful'),\n",
       " ('things', 'NNS', 'thing'),\n",
       " ('PREFACE', 'NN', 'Preface')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.tags()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'dorian': 1, 'artist': 1, 'beautiful': 1})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.postags(pos='JJ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'be': 2})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.postags(pos='VERB')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[TaggedSentence(\n",
       " \t'THE⁄The⁄DT PICTURE⁄Picture⁄NN OF⁄Of⁄IN DORIAN⁄Dorian⁄JJ GRAY⁄Gray⁄NNP BY⁄By⁄IN OSCAR⁄Oscar⁄NNP WILDE⁄Wilde⁄NNP',\n",
       " \t n=0\n",
       " ),\n",
       " TaggedSentence(\n",
       " \t'The⁄The⁄DT it⁄it⁄PRP is⁄be⁄VBZ artist⁄artist⁄JJ is⁄be⁄VBZ the⁄the⁄DT creator⁄creator⁄NN of⁄of⁄IN beautiful⁄beautiful⁄JJ things⁄thing⁄NNS',\n",
       " \t n=1\n",
       " ),\n",
       " TaggedSentence(\n",
       " \t'PREFACE⁄Preface⁄NN',\n",
       " \t n=2\n",
       " )]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Token(word='artist', idx=3, pos='JJ', lemma='artist', nsent=1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(1).tokens()[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(text1.speller)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.speller.spell('hello')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('hello', 'hellos', 'hello o', 'hellhole')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.speller.suggest('helloo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hellooя'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.speller.replace('hellooя')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('artist', 'artiste', 'artiest', 'artists')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.speller().suggest(text1.sents(1).tokens()[3].lemma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.5, 'THE PICTURE OF DORIAN GRAY BY OSCAR WILDE'),\n",
       " (0.5, 'The artist is the creator of beautiful things')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.summarize(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Token(word='The', idx=0, pos='DT', lemma='The', nsent=1),\n",
       " Token(word='it', idx=1, pos='PRP', lemma='it', nsent=1),\n",
       " Token(word='is', idx=2, pos='VBZ', lemma='be', nsent=1),\n",
       " Token(word='artist', idx=3, pos='JJ', lemma='artist', nsent=1),\n",
       " Token(word='is', idx=4, pos='VBZ', lemma='be', nsent=1),\n",
       " Token(word='the', idx=5, pos='DT', lemma='the', nsent=1),\n",
       " Token(word='creator', idx=6, pos='NN', lemma='creator', nsent=1),\n",
       " Token(word='of', idx=7, pos='IN', lemma='of', nsent=1),\n",
       " Token(word='beautiful', idx=8, pos='JJ', lemma='beautiful', nsent=1),\n",
       " Token(word='things', idx=9, pos='NNS', lemma='thing', nsent=1))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(1).tokens()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Token(word='creator', idx=6, pos='NN', lemma='creator', nsent=1),\n",
       " Token(word='things', idx=9, pos='NNS', lemma='thing', nsent=1))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(1).tokens(pos=\"NN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('the',\n",
       " 'it',\n",
       " 'is',\n",
       " 'artist',\n",
       " 'is',\n",
       " 'the',\n",
       " 'creator',\n",
       " 'of',\n",
       " 'beautiful',\n",
       " 'things')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(1).words()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('beautiful', 'artist', 'is', 'things', 'creator', 'of', 'the', 'it')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(1).words(uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The⁄The⁄DT it⁄it⁄PRP is⁄be⁄VBZ artist⁄artist⁄JJ is⁄be⁄VBZ the⁄the⁄DT creator⁄creator⁄NN of⁄of⁄IN beautiful⁄beautiful⁄JJ things⁄thing⁄NNS'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(1).text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The it is artist is the creator of beautiful things'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(1).raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['THE',\n",
       " 'PICTURE',\n",
       " 'OF',\n",
       " 'DORIAN',\n",
       " 'GRAY',\n",
       " 'BY',\n",
       " 'OSCAR',\n",
       " 'WILDE',\n",
       " 'The',\n",
       " 'it',\n",
       " 'is',\n",
       " 'artist',\n",
       " 'is',\n",
       " 'the',\n",
       " 'creator',\n",
       " 'of',\n",
       " 'beautiful',\n",
       " 'things',\n",
       " 'PREFACE']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.words(lower=False, filtrate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gray', 'oscar', 'wilde']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.words(lower=True, pos=\"NNP\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['artist',\n",
       " 'wilde',\n",
       " 'things',\n",
       " 'preface',\n",
       " 'oscar',\n",
       " 'beautiful',\n",
       " 'dorian',\n",
       " 'of',\n",
       " 'by',\n",
       " 'picture',\n",
       " 'the',\n",
       " 'creator',\n",
       " 'it',\n",
       " 'gray',\n",
       " 'is']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.words(lower=True, uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('THE', 'DT', 'The'),\n",
       " ('PICTURE', 'NN', 'Picture'),\n",
       " ('OF', 'IN', 'Of'),\n",
       " ('DORIAN', 'JJ', 'Dorian'),\n",
       " ('GRAY', 'NNP', 'Gray'),\n",
       " ('BY', 'IN', 'By'),\n",
       " ('OSCAR', 'NNP', 'Oscar'),\n",
       " ('WILDE', 'NNP', 'Wilde')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.sents(0).tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.count(words=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.count(words=True,token='the')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.count(words=False, uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dorian',\n",
       " 'the',\n",
       " 'it',\n",
       " 'thing',\n",
       " 'preface',\n",
       " 'picture',\n",
       " 'gray',\n",
       " 'wilde',\n",
       " 'by',\n",
       " 'beautiful',\n",
       " 'artist',\n",
       " 'creator',\n",
       " 'be',\n",
       " 'oscar',\n",
       " 'of']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.lemmas(uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'Picture',\n",
       " 'Of',\n",
       " 'Dorian',\n",
       " 'Gray',\n",
       " 'By',\n",
       " 'Oscar',\n",
       " 'Wilde',\n",
       " 'The',\n",
       " 'it',\n",
       " 'be',\n",
       " 'artist',\n",
       " 'be',\n",
       " 'the',\n",
       " 'creator',\n",
       " 'of',\n",
       " 'beautiful',\n",
       " 'thing',\n",
       " 'Preface']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.lemmas(lower=False, filtrate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.count(words=False,uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.count(words=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['beautiful things',\n",
       " 'dorian gray',\n",
       " 'oscar wilde',\n",
       " 'creator',\n",
       " 'preface',\n",
       " 'picture',\n",
       " 'artist']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.keywords().topn(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', (0, 0)),\n",
       " ('picture', (0, 1)),\n",
       " ('of', (0, 2)),\n",
       " ('dorian', (0, 3)),\n",
       " ('gray', (0, 4)),\n",
       " ('by', (0, 5)),\n",
       " ('oscar', (0, 6)),\n",
       " ('wilde', (0, 7)),\n",
       " ('the', (1, 0)),\n",
       " ('it', (1, 1)),\n",
       " ('is', (1, 2)),\n",
       " ('artist', (1, 3)),\n",
       " ('is', (1, 4)),\n",
       " ('the', (1, 5)),\n",
       " ('creator', (1, 6)),\n",
       " ('of', (1, 7)),\n",
       " ('beautiful', (1, 8)),\n",
       " ('things', (1, 9)),\n",
       " ('preface', (2, 0))]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.trie.items(sort=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['artist',\n",
       " 'beautiful',\n",
       " 'by',\n",
       " 'creator',\n",
       " 'dorian',\n",
       " 'gray',\n",
       " 'is',\n",
       " 'is',\n",
       " 'it',\n",
       " 'of',\n",
       " 'of',\n",
       " 'oscar',\n",
       " 'picture',\n",
       " 'preface',\n",
       " 'the',\n",
       " 'the',\n",
       " 'the',\n",
       " 'things',\n",
       " 'wilde']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.trie.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'list'>, {0: ['the', 'picture', 'of', 'dorian', 'gray', 'by', 'oscar', 'wilde'], 1: ['the', 'artist', 'is', 'the', 'creator', 'of', 'beautiful', 'things'], 2: ['preface']})\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "sents = defaultdict(list)\n",
    "for w,(nsent,idx) in trie:\n",
    "    sents[nsent].append(w)\n",
    "\n",
    "print(sents)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The⁄The⁄DT Picture⁄Picture⁄NN of⁄of⁄IN Dorian⁄Dorian⁄JJ Gray⁄Gray⁄NNP By⁄By⁄IN Oscar⁄Oscar⁄NNP Wilde⁄Wilde⁄NNP\n"
     ]
    }
   ],
   "source": [
    "intake = 'The Picture of Dorian Gray By Oscar Wilde'\n",
    "text2 = Text(intake, prep, clean, filters, input='text', inplace=True)\n",
    "print(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('The', 'DT', 'The'),\n",
       " ('Picture', 'NN', 'Picture'),\n",
       " ('of', 'IN', 'of'),\n",
       " ('Dorian', 'JJ', 'Dorian'),\n",
       " ('Gray', 'NNP', 'Gray'),\n",
       " ('By', 'IN', 'By'),\n",
       " ('Oscar', 'NNP', 'Oscar'),\n",
       " ('Wilde', 'NNP', 'Wilde')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text2.sents(0).tuples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text(\n",
      "\tname='StringIO',\n",
      "\tencoding='None',\n",
      "\tnsents=1,\n",
      "\tnwords=8,\n",
      "\tnlemmas=8\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(repr(text2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## А теперь создадим корпус из коллекции документов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method tfidf in module nlptk.mining.text:\n",
      "\n",
      "tfidf(n_doc, token=None, texts: List[List[str]] = [], sort=False, top=0) method of nlptk.mining.text.Corpus instance\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# MODULEDIR это путь до директории каталога nlptk относительно которого мы получаем путь к коллекции документов\n",
    "source = os.path.abspath(os.path.join(nlptk.MODULEDIR,r'corpus\\en'))\n",
    "corpus = Corpus(Path(source,\"*.txt\"), prep, clean, filters)\n",
    "#corpus.saveas = (\"txt\",\"pickle\") # по умолчанию сохраняется в обоих форматах\n",
    "# но загружаться будет согласно тому, что указано в loadas\n",
    "corpus.loadas = \"pickle\"\n",
    "#corpus.verbose = True  # при True будет выводить небольшую отладочную информацию об этапах обработки каждого документа\n",
    "#corpus.rewrite = True # перетегировать все тексты, даже если они сохранены \n",
    "#corpus.filtrate = True # применить фильтры при формировании общего вокабуляра; по умолчанию False\n",
    "print(help(corpus.tfidf))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "texts = []\n",
    "\n",
    "for text in corpus:\n",
    "    # можно выводить объекты текстов text и что-то с ними делать;  по умолчанию объекты  Text генераторы предложений,\n",
    "    # но класс Corpus передает им параметр inplace=True, что заставляет  их отработать на месте \n",
    "    texts.append(text) # в списке texts будут уже отработанные экзмепляры, а не генераторы\n",
    "    #for i,sent in enumerate(text):\n",
    "    #    sent # можно выводить предложения при обработке, но это замедлит работу \n",
    "        \n",
    "print('Done!')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus(\n",
      "\tnames=['Austin Pride and Prejudice.txt', 'bronte_jane_txt.txt',...],\n",
      "\tndocs=21,\n",
      "\tnwords=1923504,\n",
      "\tnlemmas=44985,\n",
      "\tnhapaxes=18123\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(repr(corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1923504"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(corpus.ccfs().values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Последнее предложение в корпусе, которое осталось в этой переменной после цикла обработки\n",
    "# сами предложения в объекте корпуса никак не сохраняются в целях экономии памяти. \n",
    "# Сохраняются только всевозможные частотные словари\n",
    "#sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(\n",
       "\tname='wilde_picture_of_dorian_gray_txt',\n",
       "\tencoding='unknown',\n",
       "\tnsents=6504,\n",
       "\tnwords=79460,\n",
       "\tnlemmas=5785\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# работаем с последним текстом, который интерпретатор сохранил в этой переменной\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = texts[0].embed.train(texts[0].sents2lemmas(filtrate=True,lower=True),size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('might', 0.9999929070472717),\n",
       " ('lydia', 0.9999927878379822),\n",
       " ('though', 0.9999927878379822),\n",
       " ('family', 0.9999927282333374),\n",
       " ('without', 0.9999927282333374),\n",
       " ('give', 0.9999925494194031),\n",
       " ('manner', 0.9999925494194031),\n",
       " ('friend', 0.9999923706054688),\n",
       " ('soon', 0.9999923706054688),\n",
       " ('father', 0.999992311000824)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['mother'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('might', 0.9999924898147583),\n",
       " ('first', 0.9999924898147583),\n",
       " ('sister', 0.9999924302101135),\n",
       " ('lydia', 0.9999923706054688),\n",
       " ('darcy', 0.999992311000824),\n",
       " ('would', 0.9999921321868896),\n",
       " ('manner', 0.9999920725822449),\n",
       " ('family', 0.9999920129776001),\n",
       " ('find', 0.9999919533729553),\n",
       " ('much', 0.9999919533729553)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['father'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "104790"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].nwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6794"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents =  texts[3].sents2words(filtrate=True, pos={\"NN\",\"NNP\",\"VB\",\"ADJ\"}, lower=True)\n",
    "sents2 = texts[3].sents2lemmas(filtrate=True,pos={\"NN\",\"NNP\",\"VB\",\"ADJ\"}, lower=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adventures', 'sherlock', 'holmes', 'arthur', 'conan', 'doyle']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "sents[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model3 = texts[3].embed.train(sents, size=1000, iter=15, window=2, sg=1, seed=10)\n",
    "model4 = texts[3].embed.train(sents, size=1000, iter=15, window=2, sg=1, seed=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('account', 0.9999289512634277),\n",
       " ('danger', 0.9999247789382935),\n",
       " ('advice', 0.9999229907989502),\n",
       " ('george', 0.9999229311943054),\n",
       " ('walk', 0.9999223947525024),\n",
       " ('come', 0.9999217391014099),\n",
       " ('show', 0.9999217391014099),\n",
       " ('box', 0.9999212026596069),\n",
       " ('evidence', 0.9999212026596069),\n",
       " ('ventilator', 0.9999203681945801)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model3.wv.most_similar(positive=['doctor'], topn=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('friend', 0.9997193813323975),\n",
       " ('surprise', 0.9997183084487915),\n",
       " ('oclock', 0.9997113943099976),\n",
       " ('chair', 0.9997076392173767),\n",
       " ('silence', 0.9997074007987976),\n",
       " ('morning', 0.9996960163116455),\n",
       " ('shoulder', 0.999691367149353),\n",
       " ('armchair', 0.9996887445449829),\n",
       " ('christmas', 0.9996850490570068),\n",
       " ('ceiling', 0.9996834397315979)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model4.wv.most_similar(positive=['holmes'], topn=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['THE⁄The⁄DT PICTURE⁄Picture⁄NN OF⁄Of⁄IN DORIAN⁄Dorian⁄JJ GRAY⁄Gray⁄NNP '\n",
      " 'BY⁄By⁄IN OSCAR⁄Oscar⁄NNP WILDE⁄Wilde⁄NNP THE⁄The⁄DT PREFACE⁄Preface⁄NNP '\n",
      " 'The⁄The⁄DT artist⁄artist⁄NN is⁄be⁄VBZ the⁄the⁄DT creator⁄creator⁄NN of⁄of⁄IN '\n",
      " 'beautiful⁄beautiful⁄JJ things⁄thing⁄NNS',\n",
      " 'To⁄To⁄TO reveal⁄reveal⁄VB art⁄art⁄NN and⁄and⁄CC conceal⁄conceal⁄VB '\n",
      " 'the⁄the⁄DT artist⁄artist⁄NN is⁄be⁄VBZ art⁄art⁄JJ aim⁄aim⁄NN',\n",
      " 'The⁄The⁄DT critic⁄critic⁄NN is⁄be⁄VBZ he⁄he⁄PRP who⁄who⁄WP can⁄can⁄MD '\n",
      " 'translate⁄translate⁄VB into⁄into⁄IN another⁄another⁄DT manner⁄manner⁄NN '\n",
      " 'or⁄or⁄CC a⁄a⁄DT new⁄new⁄JJ material⁄material⁄NN his⁄his⁄PRP$ '\n",
      " 'impression⁄impression⁄NN of⁄of⁄IN beautiful⁄beautiful⁄JJ things⁄thing⁄NNS',\n",
      " 'The⁄The⁄DT highest⁄high⁄JJS as⁄a⁄IN the⁄the⁄DT lowest⁄low⁄JJS form⁄form⁄NN '\n",
      " 'of⁄of⁄IN criticism⁄criticism⁄NN is⁄be⁄VBZ a⁄a⁄DT mode⁄mode⁄NN of⁄of⁄IN '\n",
      " 'autobiography⁄autobiography⁄NN',\n",
      " 'Those⁄Those⁄DT who⁄who⁄WP find⁄find⁄VBP ugly⁄ugly⁄JJ meanings⁄meaning⁄NNS '\n",
      " 'in⁄in⁄IN beautiful⁄beautiful⁄JJ things⁄thing⁄NNS are⁄be⁄VBP '\n",
      " 'corrupt⁄corrupt⁄JJ without⁄without⁄IN being⁄be⁄VBG charming⁄charm⁄VBG',\n",
      " 'This⁄This⁄DT is⁄be⁄VBZ a⁄a⁄DT fault⁄fault⁄NN',\n",
      " 'Those⁄Those⁄DT who⁄who⁄WP find⁄find⁄VBP beautiful⁄beautiful⁄JJ '\n",
      " 'meanings⁄meaning⁄NNS in⁄in⁄IN beautiful⁄beautiful⁄JJ things⁄thing⁄NNS '\n",
      " 'are⁄be⁄VBP the⁄the⁄DT cultivated⁄cultivated⁄JJ',\n",
      " 'For⁄For⁄IN these⁄these⁄DT there⁄there⁄EX is⁄be⁄VBZ hope⁄hope⁄NN',\n",
      " 'They⁄They⁄PRP are⁄be⁄VBP the⁄the⁄DT elect⁄elect⁄NN to⁄to⁄TO whom⁄whom⁄WP '\n",
      " 'beautiful⁄beautiful⁄JJ things⁄thing⁄NNS mean⁄mean⁄VBP only⁄only⁄RB '\n",
      " 'Beauty⁄Beauty⁄NNP',\n",
      " 'There⁄There⁄EX is⁄be⁄VBZ no⁄no⁄DT such⁄such⁄JJ thing⁄thing⁄NN as⁄a⁄IN a⁄a⁄DT '\n",
      " 'moral⁄moral⁄JJ or⁄or⁄CC an⁄an⁄DT immoral⁄immoral⁄JJ book⁄book⁄NN']\n"
     ]
    }
   ],
   "source": [
    "# объект text при печати на консоль отображается как текст аналогичный сохраненному на диске тегированному тексту\n",
    "pprint(str(text).split(\"\\n\")[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34895"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# получаем общее числу слов в тексте, но после фильтрации\n",
    "len(text.words(filtrate=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79460"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text.words(filtrate=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79460"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# число слов в тексте без фильтрации\n",
    "text.nwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7049"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text.count(words=True, uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5785"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text.count(words=False, uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79460"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text.trie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1923504"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# считаем сколько всего слов есть в корпусе по всем документам\n",
    "sum(corpus.ccfs().values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ПРОВЕРЯЕМ ЧТО ПОДСЧЕТЫ ВХОЖДЕНИЙ СЛОВ ОДИНАКОВЫ КАК ДЛЯ ОБЪЕКТОВ ТЕКСТОВ, ТАК И НА УРОВНЕ ВОКАБУЛЯРА КОРПУСА"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Austin Pride and Prejudice.txt', 121779),\n",
       " ('bronte_jane_txt.txt', 187333),\n",
       " ('bronte_wuthering_txt.txt', 118103),\n",
       " ('doyle_the_adventures.txt', 104790),\n",
       " ('dreiser_sister.txt', 155595),\n",
       " ('Edgar Allan Poe The Cask of Amontillado.txt', 2333),\n",
       " ('Edgar Allan Poe The Masque of the Red Death.txt', 2415),\n",
       " ('Edgar Allan Poe The Tell-Tale Heart.txt', 2119),\n",
       " ('fitzgerald_great_gatsby_txt.txt', 49346),\n",
       " ('Franz Kafka - Metamorphosis.txt', 22167),\n",
       " ('John Steinbeck - Of Mice and Men.txt', 30743),\n",
       " ('kipling_jungle_book_txt.txt', 51379),\n",
       " ('london_white_txt.txt', 72421),\n",
       " ('stevenson_treasure_island_txt.txt', 69182),\n",
       " ('Stivenson_Fall-or-Dodge-in-Hell_RuLit_Me.txt', 318367),\n",
       " ('stoker_dracula_txt.txt', 160645),\n",
       " ('twain_tom_sawyer_txt.txt', 72251),\n",
       " ('walter_scott_ivanhoe_txt.txt', 193243),\n",
       " ('wells_invisible_man_txt.txt', 49434),\n",
       " ('wells_war_of_the_worlds_txt.txt', 60399),\n",
       " ('wilde_picture_of_dorian_gray_txt.txt', 79460)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(txt.filename,txt.nwords) for txt in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Austin Pride and Prejudice.txt', 121779),\n",
       " ('bronte_jane_txt.txt', 187333),\n",
       " ('bronte_wuthering_txt.txt', 118103),\n",
       " ('doyle_the_adventures.txt', 104790),\n",
       " ('dreiser_sister.txt', 155595),\n",
       " ('Edgar Allan Poe The Cask of Amontillado.txt', 2333),\n",
       " ('Edgar Allan Poe The Masque of the Red Death.txt', 2415),\n",
       " ('Edgar Allan Poe The Tell-Tale Heart.txt', 2119),\n",
       " ('fitzgerald_great_gatsby_txt.txt', 49346),\n",
       " ('Franz Kafka - Metamorphosis.txt', 22167),\n",
       " ('John Steinbeck - Of Mice and Men.txt', 30743),\n",
       " ('kipling_jungle_book_txt.txt', 51379),\n",
       " ('london_white_txt.txt', 72421),\n",
       " ('stevenson_treasure_island_txt.txt', 69182),\n",
       " ('Stivenson_Fall-or-Dodge-in-Hell_RuLit_Me.txt', 318367),\n",
       " ('stoker_dracula_txt.txt', 160645),\n",
       " ('twain_tom_sawyer_txt.txt', 72251),\n",
       " ('walter_scott_ivanhoe_txt.txt', 193243),\n",
       " ('wells_invisible_man_txt.txt', 49434),\n",
       " ('wells_war_of_the_worlds_txt.txt', 60399),\n",
       " ('wilde_picture_of_dorian_gray_txt.txt', 79460)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# считаем число слов для каждого текст по сумме частот из каждого словаря cfs\n",
    "# это должно совпасть с тем числом, которое каждый Text хранит в свойстве nwords\n",
    "[(name, sum(corpus.cfs(n).values())) for n,name in enumerate(corpus.filenames())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TaggedSentence(\n",
       "\t'THE⁄The⁄DT PICTURE⁄Picture⁄NN OF⁄Of⁄IN DORIAN⁄Dorian⁄JJ GRAY⁄Gray⁄NNP BY⁄By⁄IN OSCAR⁄Oscar⁄NNP WILDE⁄Wilde⁄NNP THE⁄The⁄DT PREFACE⁄Preface⁄NNP The⁄The⁄DT artist⁄artist⁄NN is⁄be⁄VBZ the⁄the⁄DT creator⁄creator⁄NN of⁄of⁄IN beautiful⁄beautiful⁄JJ things⁄thing⁄NNS',\n",
       "\t n=0\n",
       ")"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# объект предложения в виде экземпляра класса TaggedSentence; n это порядковый номер предложения\n",
    "text.sents(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n",
      "18\n",
      "18\n",
      "14\n",
      "14\n"
     ]
    }
   ],
   "source": [
    "# легко догадаться, что это число слов в предложении\n",
    "print(text.sents(0).nwords)\n",
    "print(len(text.sents(0).tokens()))\n",
    "print(len(text.sents(0).words()))\n",
    "print(len(text.sents(0).words(uniq=True)))\n",
    "print(len(text.sents(0).lemmas(uniq=True)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[TaggedSentence(\n",
       " \t'Hallward⁄NN⁄Hallward',\n",
       " \t n=177\n",
       " ), TaggedSentence(\n",
       " \t'Gray⁄NN⁄Gray',\n",
       " \t n=212\n",
       " ), TaggedSentence(\n",
       " \t'Harry⁄NNP⁄Harry',\n",
       " \t n=236\n",
       " )]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# а так мы получаем все предложения из текста не длиннее одного слова\n",
    "text.sents(max_words=1)[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[TaggedSentence(\n",
       " \t'This⁄This⁄DT is⁄be⁄VBZ a⁄a⁄DT fault⁄fault⁄NN',\n",
       " \t n=5\n",
       " ), TaggedSentence(\n",
       " \t'For⁄For⁄IN these⁄these⁄DT there⁄there⁄EX is⁄be⁄VBZ hope⁄hope⁄NN',\n",
       " \t n=7\n",
       " ), TaggedSentence(\n",
       " \t'That⁄That⁄DT is⁄be⁄VBZ all⁄all⁄DT',\n",
       " \t n=11\n",
       " )]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Давайте усложним условия: не менее трех, но не более пяти слов\n",
    "text.sents(min_words=3,max_words=5)[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Token(word='the', idx=0, pos='DT', lemma='the', nsent=0),\n",
       " Token(word='picture', idx=1, pos='NN', lemma='picture', nsent=0),\n",
       " Token(word='of', idx=2, pos='IN', lemma='of', nsent=0),\n",
       " Token(word='dorian', idx=3, pos='JJ', lemma='dorian', nsent=0),\n",
       " Token(word='gray', idx=4, pos='NNP', lemma='gray', nsent=0),\n",
       " Token(word='by', idx=5, pos='IN', lemma='by', nsent=0),\n",
       " Token(word='oscar', idx=6, pos='NNP', lemma='oscar', nsent=0),\n",
       " Token(word='wilde', idx=7, pos='NNP', lemma='wilde', nsent=0),\n",
       " Token(word='the', idx=8, pos='DT', lemma='the', nsent=0),\n",
       " Token(word='preface', idx=9, pos='NNP', lemma='preface', nsent=0),\n",
       " Token(word='the', idx=10, pos='DT', lemma='the', nsent=0),\n",
       " Token(word='artist', idx=11, pos='NN', lemma='artist', nsent=0),\n",
       " Token(word='is', idx=12, pos='VBZ', lemma='be', nsent=0),\n",
       " Token(word='the', idx=13, pos='DT', lemma='the', nsent=0),\n",
       " Token(word='creator', idx=14, pos='NN', lemma='creator', nsent=0),\n",
       " Token(word='of', idx=15, pos='IN', lemma='of', nsent=0),\n",
       " Token(word='beautiful', idx=16, pos='JJ', lemma='beautiful', nsent=0),\n",
       " Token(word='things', idx=17, pos='NNS', lemma='thing', nsent=0))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# А если мы хотим получить предложение в более информативном виде? Класс Token к вашим услугам.\n",
    "# Метод tokens трансформирует предложение в список экземпляров этого класса.\n",
    "text.sents(0).tokens(lower=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Token(word='picture', idx=1, pos='NN', lemma='picture', nsent=0),\n",
       " Token(word='dorian', idx=3, pos='JJ', lemma='dorian', nsent=0),\n",
       " Token(word='gray', idx=4, pos='NNP', lemma='gray', nsent=0),\n",
       " Token(word='oscar', idx=6, pos='NNP', lemma='oscar', nsent=0),\n",
       " Token(word='wilde', idx=7, pos='NNP', lemma='wilde', nsent=0),\n",
       " Token(word='preface', idx=9, pos='NNP', lemma='preface', nsent=0),\n",
       " Token(word='artist', idx=11, pos='NN', lemma='artist', nsent=0),\n",
       " Token(word='creator', idx=14, pos='NN', lemma='creator', nsent=0),\n",
       " Token(word='beautiful', idx=16, pos='JJ', lemma='beautiful', nsent=0),\n",
       " Token(word='things', idx=17, pos='NNS', lemma='thing', nsent=0))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Попробуем включить фильтрацию (это те самые rules_filter, которые мы передавали в объект Corpus, \n",
    "# но которые по умолчанию отключены параметром corpus.filtrate=False, \n",
    "# чтобы в объекте корпуса сохранялась статистика по всем словам\n",
    "text.sents(0).tokens(lower=True,filtrate=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('the',\n",
       " 'picture',\n",
       " 'of',\n",
       " 'dorian',\n",
       " 'gray',\n",
       " 'by',\n",
       " 'oscar',\n",
       " 'wilde',\n",
       " 'the',\n",
       " 'preface',\n",
       " 'the',\n",
       " 'artist',\n",
       " 'is',\n",
       " 'the',\n",
       " 'creator',\n",
       " 'of',\n",
       " 'beautiful',\n",
       " 'things')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# все слова предложения с указанным номером\n",
    "text.sents(0).words()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('the',\n",
       " 'picture',\n",
       " 'of',\n",
       " 'dorian',\n",
       " 'gray',\n",
       " 'by',\n",
       " 'oscar',\n",
       " 'wilde',\n",
       " 'the',\n",
       " 'preface',\n",
       " 'the',\n",
       " 'artist',\n",
       " 'be',\n",
       " 'the',\n",
       " 'creator',\n",
       " 'of',\n",
       " 'beautiful',\n",
       " 'thing')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# все леммы предложения с указанным номером\n",
    "text.sents(0).lemmas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('THE', 'DT', 'The'),\n",
       " ('PICTURE', 'NN', 'Picture'),\n",
       " ('OF', 'IN', 'Of'),\n",
       " ('DORIAN', 'JJ', 'Dorian'),\n",
       " ('GRAY', 'NNP', 'Gray'),\n",
       " ('BY', 'IN', 'By'),\n",
       " ('OSCAR', 'NNP', 'Oscar'),\n",
       " ('WILDE', 'NNP', 'Wilde'),\n",
       " ('THE', 'DT', 'The'),\n",
       " ('PREFACE', 'NNP', 'Preface'),\n",
       " ('The', 'DT', 'The'),\n",
       " ('artist', 'NN', 'artist'),\n",
       " ('is', 'VBZ', 'be'),\n",
       " ('the', 'DT', 'the'),\n",
       " ('creator', 'NN', 'creator'),\n",
       " ('of', 'IN', 'of'),\n",
       " ('beautiful', 'JJ', 'beautiful'),\n",
       " ('things', 'NNS', 'thing')]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# разтегированное представаление предложения\n",
    "text.sents(0).tuples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('picture', 'artist', 'creator', 'things')"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# слова предложения отфильтрованные по частям речи - сущ. (в ед. и мн. числе)\n",
    "text.sents(0).words(pos={\"NN\",\"NNS\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('picture', 'artist', 'creator', 'thing')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# леммы предложения отфильтованные по частям речи - сущ. (в ед. и мн. числе)\n",
    "text.sents(0).lemmas(pos={\"NN\",\"NNS\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('picture', 'gray', 'oscar', 'wilde', 'preface', 'artist', 'creator', 'thing')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# все леммы чей postag начинается на N, то есть любые существительные\n",
    "text.sents(0).lemmas(pos=\"N\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('dorian', 'beautiful')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text.sents(0).lemmas(pos=\"JJ\")\n",
    "# имя dorian неверно определяется как прилагательное, но здесь играет роль неоднозначность слова, так как такое прилагательное \n",
    "# тоже существует в английском"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'THE PICTURE OF DORIAN GRAY BY OSCAR WILDE THE PREFACE The artist is the creator of beautiful things'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Предложение как есть, но уже без пунктуации, так как она была удалена на этапе очистки текста фильтрами TextCleaner\n",
    "text.sents(0).raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['the',\n",
       " 'picture',\n",
       " 'of',\n",
       " 'dorian',\n",
       " 'gray',\n",
       " 'by',\n",
       " 'oscar',\n",
       " 'wilde',\n",
       " 'the',\n",
       " 'preface']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text.words(filtrate=False, pos=\"N\")[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'be': 3566, 'have': 1587, 'do': 656, 'say': 388, 'go': 295, 'know': 261, 'come': 232, 'look': 212, 'make': 205, 'think': 199, ...})"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Можно получить частотные словари по любой части речи\n",
    "verbs = text.postags(\"VERB\")\n",
    "verbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('be', 3566),\n",
       " ('have', 1587),\n",
       " ('do', 656),\n",
       " ('say', 388),\n",
       " ('go', 295),\n",
       " ('know', 261),\n",
       " ('come', 232),\n",
       " ('look', 212),\n",
       " ('make', 205),\n",
       " ('think', 199)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# отсортировать по убыванию частоты\n",
    "text.postags(\"VERB\", sort=-1)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['be', 'have', 'do', 'say', 'go', 'know', 'come', 'look', 'make', 'think']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# выбрать top наиболее частотных \n",
    "text.postags(\"VERB\",top=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#verbs.hapaxes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'dorian': 292, 'good': 138, 'own': 130, 'great': 82, 'little': 82, 'young': 78, 'other': 72, 'more': 65, 'such': 65, 'old': 62, ...})"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adjectives = text.postags(\"ADJ\")\n",
    "adjectives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "possessive = text.postags(\"POS\",sort=-1)[:10]\n",
    "possessive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('picture', 'dorian', 'gray')\n",
      "('picture', 'dorian', 'oscar')\n",
      "('picture', 'dorian', 'wilde')\n",
      "('picture', 'gray', 'oscar')\n",
      "('picture', 'gray', 'wilde')\n",
      "('picture', 'oscar', 'wilde')\n",
      "('dorian', 'gray', 'oscar')\n",
      "('dorian', 'gray', 'wilde')\n",
      "('dorian', 'gray', 'preface')\n",
      "('dorian', 'oscar', 'wilde')\n",
      "('dorian', 'oscar', 'preface')\n",
      "('dorian', 'wilde', 'preface')\n"
     ]
    }
   ],
   "source": [
    "# генерация skipgram из текста (имплементация из nltk)\n",
    "for n,s in enumerate(text.skipgrams(3,2, filtrate=True, words=True)):\n",
    "    print(s)\n",
    "    if n > 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('<s>', '<s>', 'The')\n",
      "('<s>', 'The', 'Picture')\n",
      "('The', 'Picture', 'Of')\n",
      "('Picture', 'Of', 'Dorian')\n",
      "('Of', 'Dorian', 'Gray')\n",
      "('Dorian', 'Gray', 'By')\n",
      "('Gray', 'By', 'Oscar')\n",
      "('By', 'Oscar', 'Wilde')\n",
      "('Oscar', 'Wilde', 'The')\n",
      "('Wilde', 'The', 'Preface')\n",
      "('The', 'Preface', 'The')\n",
      "('Preface', 'The', 'artist')\n"
     ]
    }
   ],
   "source": [
    "# генерация ngram из текста (имплементация из nltk)\n",
    "for n,s in enumerate(text.ngrams(3, lower=False, pad_left=True, left_pad_symbol='<s>', pad_right=True, right_pad_symbol='</s>')):\n",
    "    print(s)\n",
    "    if n > 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 401),\n",
       " (1, 583),\n",
       " (2, 375),\n",
       " (3, 486),\n",
       " (4, 1474),\n",
       " (5, 24),\n",
       " (6, 0),\n",
       " (7, 2),\n",
       " (8, 236),\n",
       " (9, 51),\n",
       " (10, 390),\n",
       " (11, 430),\n",
       " (12, 70),\n",
       " (13, 341),\n",
       " (14, 1289),\n",
       " (15, 567),\n",
       " (16, 356),\n",
       " (17, 1451),\n",
       " (18, 531),\n",
       " (19, 166),\n",
       " (20, 262)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#[(n,corpus.cfs(n).get('said',0)) for n in range(corpus.ndocs)]\n",
    "[(n,corpus.cfs(n,'said')) for n in range(corpus.ndocs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.0001606587000381203),\n",
       " (1, 0.00015184012272679605),\n",
       " (2, 0.00015491826256349981),\n",
       " (3, 0.00022628132251497257),\n",
       " (4, 0.00046220445377899574),\n",
       " (5, 0.000501913390512803),\n",
       " (6, 0),\n",
       " (7, 4.6050178545948135e-05),\n",
       " (8, 0.00023334168410784996),\n",
       " (9, 0.00011225237391803286),\n",
       " (10, 0.0006189429797377777),\n",
       " (11, 0.0004083335719429296),\n",
       " (12, 4.7159131907323064e-05),\n",
       " (13, 0.00024048807466936964),\n",
       " (14, 0.0001975409562372919),\n",
       " (15, 0.0001722059390834945),\n",
       " (16, 0.00024040218743433045),\n",
       " (17, 0.00036634976795974964),\n",
       " (18, 0.000524084176355715),\n",
       " (19, 0.000134094393154286),\n",
       " (20, 0.0001608736850288346)]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(n,corpus.tfidf(n,'said')) for n in range(corpus.ndocs)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3,\n",
       " [('holmes', 0.013422744214411884),\n",
       "  ('sherlock', 0.0021765760086349586),\n",
       "  ('watson', 0.0015041389643427844),\n",
       "  ('lestrade', 0.0011040352384148304),\n",
       "  ('rucastle', 0.0011040352384148304),\n",
       "  ('mccarthy', 0.0009587674438865632),\n",
       "  ('simon', 0.0008975571169628696)],\n",
       " 'doyle_the_adventures.txt')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# набор наиболее весомых по метрике TFIDF слов в каждом тексте\n",
    "res = [(n,\n",
    "        corpus.tfidf(n,sort=-1)[0:7],\n",
    "        name\n",
    "       ) \n",
    "     for n,name in enumerate(corpus.filenames())\n",
    "]\n",
    "# берем текст по индексу 3 - Приключения Шерлока Холмса\n",
    "res[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# отображение слов из 3-го текста в порядковый номер в словаре tdidf\n",
    "ordered_by_tfidf = {word:n for n,(word,_) in enumerate(corpus.tfidf(3,sort=-1))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ordered_by_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# как видим, хотя метрика и дает наибольшие веса словам важным для контекста произведения - именам главных персонажей,\n",
    "# но очень быстро скатывается до уровня частоупотребляемых вместе с этими словами сокращений типа mr.\n",
    "# чья документтная частота очень и очень высока \n",
    "# (но их можно удалять фильтрацией, которую мы на уровне корпуса не включали, поэтому все стоп-слова и остались на месте)\n",
    "\n",
    "# документная частота слова\n",
    "corpus.dfs('mr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# \n",
    "ordered_by_tfidf['mr'] # на  8 месте если упоядочить слова по TFIDF "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "83"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered_by_tfidf['said']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# число вхождений в текст по индексу 3\n",
    "corpus.cfs(3,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# документная частота слова \"holmes\"\n",
    "corpus.dfs('holmes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "462\n",
      "462\n",
      "462\n",
      "462\n"
     ]
    }
   ],
   "source": [
    "# число вхождений в текст по индексу 3\n",
    "print(corpus.cfs(3,'holmes'))   # число вхождений словоформы\n",
    "print(texts[3].vocab['holmes']) # число вхождений лемм\n",
    "print(texts[3].words(filtrate=False).count('holmes'))\n",
    "print(texts[3].lemmas(filtrate=False).count('holmes'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n",
      "3\n",
      "15\n",
      "12\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "print(corpus.cfs(3,'investigation'))       # число вхождений словоформы\n",
    "print(corpus.cfs(3,'investigations'))\n",
    "print(texts[3].vocab.get('investigation')) # число вхождений леммы == числу вхождений словоформ, которые образуют данную лемму\n",
    "print(texts[3].words(filtrate=False).count('investigation'))  # число вхождений словоформы\n",
    "print(texts[3].lemmas(filtrate=False).count('investigation')) # число вхождений леммы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6639"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# уникальных лемм\n",
    "texts[3].count(words=False,uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8132"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# уникальных слов (без учета регистра)\n",
    "texts[3].count(words=True, uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8724"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# уникальных слов (c учетом регистра)\n",
    "texts[3].count(words=True, uniq=True, lower=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "104790"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# всего слов с повторами вхождений\n",
    "texts[3].count(words=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "104790"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].nwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "104790"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# число ключей в дереве == числу всех слов с повторами\n",
    "len(texts[3].trie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8132"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(texts[3].words(lower=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8724"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(texts[3].words(lower=False)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00020721212951756239"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.tfidf(6,\"red\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "486"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.cfs(3,'said')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.dfs('said')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00022664684706076194"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.tfidf(3,'said')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('prospero', 0.007551523202290425),\n",
       " ('courtiers', 0.0038881773578561016),\n",
       " ('waltzers', 0.0037757616011452125),\n",
       " ('mummer', 0.0037757616011452125),\n",
       " ('prince', 0.0036805490437174567),\n",
       " ('ebony', 0.0029662764061375003),\n",
       " ('musicians', 0.002413282532933419),\n",
       " ('revellers', 0.002413282532933419),\n",
       " ('suite', 0.002071538600240377),\n",
       " ('sable', 0.0020565044356389405)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.tfidf(6,sort=-1)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['prospero',\n",
       " 'courtiers',\n",
       " 'waltzers',\n",
       " 'mummer',\n",
       " 'prince',\n",
       " 'ebony',\n",
       " 'musicians',\n",
       " 'revellers',\n",
       " 'suite',\n",
       " 'sable']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# тоже самое (только в виде упорядоченного списка) - с использованием параметра top\n",
    "corpus.tfidf(6,top=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method tfidf in module nlptk.mining.text:\n",
      "\n",
      "tfidf(n_doc, texts: List[List[str]] = None, token=None, sort=False) method of nlptk.mining.text.Corpus instance\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(corpus.tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "317\n",
      "186971\n",
      "1\n",
      "0.0016954500965390355\n",
      "3.044522437723423\n",
      "0.005161835860953437\n"
     ]
    }
   ],
   "source": [
    "print(corpus.cfs(1,'rochester'))\n",
    "print(sum(corpus.cfs(1).values()))\n",
    "print(corpus.dfs('rochester'))\n",
    "print(corpus.tf(1,'rochester'))\n",
    "print(corpus.idf('rochester'))\n",
    "print(corpus.tfidf(1,token='rochester'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--cfc--\n",
      "[('the', 4320),\n",
      " ('to', 4127),\n",
      " ('of', 3596),\n",
      " ('and', 3529),\n",
      " ('her', 2216),\n",
      " ('i', 2046),\n",
      " ('a', 1945),\n",
      " ('in', 1861),\n",
      " ('was', 1843),\n",
      " ('she', 1703)]\n",
      "--ccfc--\n",
      "[('the', 104005),\n",
      " ('and', 64431),\n",
      " ('of', 53145),\n",
      " ('to', 51249),\n",
      " ('a', 42386),\n",
      " ('i', 34454),\n",
      " ('in', 29525),\n",
      " ('he', 28953),\n",
      " ('was', 26894),\n",
      " ('it', 24706)]\n",
      "--dfc--\n",
      "[('over', 21),\n",
      " ('once', 21),\n",
      " ('moment', 21),\n",
      " ('a', 21),\n",
      " ('minutes', 21),\n",
      " ('by', 21),\n",
      " ('four', 21),\n",
      " ('not', 21),\n",
      " ('from', 21),\n",
      " ('loud', 21)]\n",
      "--tfidf--\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input(\"--cfc--\")\n",
    "pprint(corpus.cfs(0,sort=-1)[:10])\n",
    "input(\"--ccfc--\")\n",
    "pprint(corpus.ccfs(sort=-1)[:10])\n",
    "input(\"--dfc--\")\n",
    "pprint(corpus.dfs(sort=-1)[:10])\n",
    "input(\"--tfidf--\")\n",
    "\n",
    "#for n in range(corpus.ndocs):\n",
    "#    pprint(corpus.tfidf(n,sort=-1)[:10]) \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18114"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# общее число слов-одиночек\n",
    "all_hapaxes = corpus.hapaxes()\n",
    "len(all_hapaxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a-bleeding',\n",
       " 'a-blowing',\n",
       " 'a-buttin',\n",
       " 'a-callin',\n",
       " 'a-chaffin',\n",
       " 'a-changing',\n",
       " 'a-crossin',\n",
       " 'a-doin',\n",
       " 'a-doing',\n",
       " 'a-done']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# н-да, вот что значит не включать фильтрацию на корпусе ...\n",
    "sorted(all_hapaxes)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['austen',\n",
       " 'rightful',\n",
       " 'morris',\n",
       " 'grown-up',\n",
       " 'newcomers',\n",
       " 'over-scrupulous',\n",
       " 'vexing',\n",
       " 'sarcastic',\n",
       " 'develop',\n",
       " 'solace']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# получаем слова-одиночки для каждого текста в корпусе\n",
    "hapaxes = [(path, corpus.hapaxes(n)) for n,path in enumerate(corpus.filenames())]\n",
    "hapaxes[0][1][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Austin Pride and Prejudice.txt', 2476),\n",
       " ('bronte_jane_txt.txt', 5809),\n",
       " ('bronte_wuthering_txt.txt', 4281),\n",
       " ('doyle_the_adventures.txt', 3775),\n",
       " ('dreiser_sister.txt', 4567),\n",
       " ('Edgar Allan Poe The Cask of Amontillado.txt', 545),\n",
       " ('Edgar Allan Poe The Masque of the Red Death.txt', 585),\n",
       " ('Edgar Allan Poe The Tell-Tale Heart.txt', 384),\n",
       " ('fitzgerald_great_gatsby_txt.txt', 3200),\n",
       " ('Franz Kafka - Metamorphosis.txt', 1295),\n",
       " ('John Steinbeck - Of Mice and Men.txt', 1397),\n",
       " ('kipling_jungle_book_txt.txt', 2211),\n",
       " ('london_white_txt.txt', 3205),\n",
       " ('stevenson_treasure_island_txt.txt', 2971),\n",
       " ('Stivenson_Fall-or-Dodge-in-Hell_RuLit_Me.txt', 8757),\n",
       " ('stoker_dracula_txt.txt', 4487),\n",
       " ('twain_tom_sawyer_txt.txt', 3707),\n",
       " ('walter_scott_ivanhoe_txt.txt', 5791),\n",
       " ('wells_invisible_man_txt.txt', 2927),\n",
       " ('wells_war_of_the_worlds_txt.txt', 3333),\n",
       " ('wilde_picture_of_dorian_gray_txt.txt', 3543)]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# сколько у нас гапаксов на каждый текст\n",
    "[(h[0],len(h[1])) for h in hapaxes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Austin Pride and Prejudice.txt', 2476),\n",
       " ('bronte_jane_txt.txt', 5809),\n",
       " ('bronte_wuthering_txt.txt', 4281),\n",
       " ('doyle_the_adventures.txt', 3775),\n",
       " ('dreiser_sister.txt', 4567),\n",
       " ('Edgar Allan Poe The Cask of Amontillado.txt', 545),\n",
       " ('Edgar Allan Poe The Masque of the Red Death.txt', 585),\n",
       " ('Edgar Allan Poe The Tell-Tale Heart.txt', 384),\n",
       " ('fitzgerald_great_gatsby_txt.txt', 3200),\n",
       " ('Franz Kafka - Metamorphosis.txt', 1295),\n",
       " ('John Steinbeck - Of Mice and Men.txt', 1397),\n",
       " ('kipling_jungle_book_txt.txt', 2211),\n",
       " ('london_white_txt.txt', 3205),\n",
       " ('stevenson_treasure_island_txt.txt', 2971),\n",
       " ('Stivenson_Fall-or-Dodge-in-Hell_RuLit_Me.txt', 8757),\n",
       " ('stoker_dracula_txt.txt', 4487),\n",
       " ('twain_tom_sawyer_txt.txt', 3707),\n",
       " ('walter_scott_ivanhoe_txt.txt', 5791),\n",
       " ('wells_invisible_man_txt.txt', 2927),\n",
       " ('wells_war_of_the_worlds_txt.txt', 3333),\n",
       " ('wilde_picture_of_dorian_gray_txt.txt', 3543)]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# гапаксы с вычислением по всем словоформам, результат должен быть идентичен коду выше \n",
    "[(txt.filename, len(txt.hapaxes(words=True))) for txt in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Austin Pride and Prejudice.txt', 1843),\n",
       " ('bronte_jane_txt.txt', 4608),\n",
       " ('bronte_wuthering_txt.txt', 3175),\n",
       " ('doyle_the_adventures.txt', 2979),\n",
       " ('dreiser_sister.txt', 3634),\n",
       " ('Edgar Allan Poe The Cask of Amontillado.txt', 486),\n",
       " ('Edgar Allan Poe The Masque of the Red Death.txt', 498),\n",
       " ('Edgar Allan Poe The Tell-Tale Heart.txt', 339),\n",
       " ('fitzgerald_great_gatsby_txt.txt', 2575),\n",
       " ('Franz Kafka - Metamorphosis.txt', 998),\n",
       " ('John Steinbeck - Of Mice and Men.txt', 1097),\n",
       " ('kipling_jungle_book_txt.txt', 1705),\n",
       " ('london_white_txt.txt', 2525),\n",
       " ('stevenson_treasure_island_txt.txt', 2269),\n",
       " ('Stivenson_Fall-or-Dodge-in-Hell_RuLit_Me.txt', 7255),\n",
       " ('stoker_dracula_txt.txt', 3537),\n",
       " ('twain_tom_sawyer_txt.txt', 2875),\n",
       " ('walter_scott_ivanhoe_txt.txt', 4502),\n",
       " ('wells_invisible_man_txt.txt', 2327),\n",
       " ('wells_war_of_the_worlds_txt.txt', 2680),\n",
       " ('wilde_picture_of_dorian_gray_txt.txt', 2808)]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# а так гапаксы будут считаться по леммам и, поскольку словарь лемм для каждого текста уже имеется в экземпляре каждого текста, \n",
    "# вычисляется гораздо быстрее\n",
    "[(txt.filename, len(txt.hapaxes())) for txt in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104790\n",
      "3775\n",
      "['weedy', 'weekly', 'weigh', 'weighing', 'weird', 'welcome', 'welcomed', 'well-cut', 'well-groomed', 'well-lit', 'well-nigh', 'well-nurtured', 'well-opened', 'well-remembered', 'wellington', 'westaway', 'westawayous', 'westbury', 'western', 'westphail', 'westward', 'wheal', 'wheel', 'wheeled', 'wherever', 'whims', 'whine', 'whined', 'whirling', 'whishing', 'whisper', 'whispering', 'white-aproned', 'white-counterpaned', 'whiten', 'whiter', 'whither', 'whittington', 'wholesome', 'whoso', 'wicker-work', 'wicket', 'widow', 'wig', 'wight', 'wigmore', 'wigs', 'wilderness', 'wilful', 'wilhelm', 'will-o-the-wisp', 'willingly', 'willows', 'wilton', 'wimpole', 'win', 'winced', 'wincing', 'wind-swept', 'windfall', 'window-sill', 'windowsill', 'wine', 'wine-cellar', 'wines', 'winking', 'winter', 'wintry', 'wiry', 'wisdom', 'wishing', 'withdrawn', 'witnesses', 'wits', 'wives', 'woke', 'womanhood', 'won', 'wondered', 'wooded', 'wooden-leg', 'wooden-legged', 'worker', 'workmen', 'world-wide', 'worlds', 'worm-eaten', 'worms', 'worry', 'worthless', 'wounded', 'wrack', 'wreath', 'wreaths', 'wrenching', 'wretch', 'wriggled', 'writ', 'writers', 'writes', 'writings', 'x', 'ycuea', 'yell', 'yonder', 'zealand', 'zero', 'zero-point', 'zest', 'zigzag']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ndoc = 3\n",
    "print(len(texts[ndoc].words(filtrate=False)))\n",
    "\n",
    "hp = texts[ndoc].hapaxes(words=True)\n",
    "print(len(hp))\n",
    "print(sorted(hp)[-110:len(hp)])\n",
    "'ycuea' in hp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104790\n",
      "3775\n",
      "['weedy', 'weekly', 'weigh', 'weighing', 'weird', 'welcome', 'welcomed', 'well-cut', 'well-groomed', 'well-lit', 'well-nigh', 'well-nurtured', 'well-opened', 'well-remembered', 'wellington', 'westaway', 'westawayous', 'westbury', 'western', 'westphail', 'westward', 'wheal', 'wheel', 'wheeled', 'wherever', 'whims', 'whine', 'whined', 'whirling', 'whishing', 'whisper', 'whispering', 'white-aproned', 'white-counterpaned', 'whiten', 'whiter', 'whither', 'whittington', 'wholesome', 'whoso', 'wicker-work', 'wicket', 'widow', 'wig', 'wight', 'wigmore', 'wigs', 'wilderness', 'wilful', 'wilhelm', 'will-o-the-wisp', 'willingly', 'willows', 'wilton', 'wimpole', 'win', 'winced', 'wincing', 'wind-swept', 'windfall', 'window-sill', 'windowsill', 'wine', 'wine-cellar', 'wines', 'winking', 'winter', 'wintry', 'wiry', 'wisdom', 'wishing', 'withdrawn', 'witnesses', 'wits', 'wives', 'woke', 'womanhood', 'won', 'wondered', 'wooded', 'wooden-leg', 'wooden-legged', 'worker', 'workmen', 'world-wide', 'worlds', 'worm-eaten', 'worms', 'worry', 'worthless', 'wounded', 'wrack', 'wreath', 'wreaths', 'wrenching', 'wretch', 'wriggled', 'writ', 'writers', 'writes', 'writings', 'x', 'ycuea', 'yell', 'yonder', 'zealand', 'zero', 'zero-point', 'zest', 'zigzag']\n"
     ]
    }
   ],
   "source": [
    "print(sum(corpus.cfs(ndoc).values()))\n",
    "ln = len(hapaxes[ndoc][1])\n",
    "print(ln)\n",
    "print(sorted(hapaxes[ndoc][1])[-110:ln])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].vocab['conan']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['conan',\n",
       " 'doyle',\n",
       " 'x',\n",
       " 'eclipses',\n",
       " 'predominates',\n",
       " 'sex',\n",
       " 'emotions',\n",
       " 'abhorrent',\n",
       " 'balanced',\n",
       " 'softer']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hapaxes[3][1][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.cfs(3,'manifested')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# общекорпусная частота слова\n",
    "corpus.ccfs('manifested')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "842\n",
      "842\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(corpus.cfs(0,'have'))\n",
    "print(texts[0].words(filtrate=False).count('have'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(\n",
       "\tname='Austin Pride and Prejudice',\n",
       "\tencoding='Windows-1252',\n",
       "\tnsents=5947,\n",
       "\tnwords=121779,\n",
       "\tnlemmas=4958\n",
       ")"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "121779"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(corpus.cfs(0).values())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3327"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.ccfs(\"am\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# частота слова для текста корпуса с номером 0\n",
    "corpus.cfs(0,'vicious')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "298"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.cfs(9,\"gregor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.ccfs(\"s\") # остатки от сокращений имен и названий у которых была удалена точка токенайзером"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.dfs('s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FreqDist({})\n"
     ]
    }
   ],
   "source": [
    "# тегов притяжательных окончаний нет, так как nltk.tag_pos умеет их определять только в виде \"'s\", а всю пунктуацию \n",
    "# в начале и в конце токена стрипает токенайзер (точнее, обертка вокруг него), поэтому в тексте остаются только токены вида \"s\"\n",
    "pos,_ = texts[10].postags(\"POS\")\n",
    "pprint(pos)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Gregor', 'NNP'), (\"'s\", 'POS')]\n",
      "[('Gregor', 'NNP'), ('s', 'NN')]\n",
      "[(\"Gregor's\", 'NN')]\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "print(nltk.pos_tag([\"Gregor\", \"'s\"]))\n",
    "#[('Gregor', 'NNP'), (\"'s\", 'POS')]\n",
    "\n",
    "print(nltk.pos_tag([\"Gregor\", \"s\"]))\n",
    "#[('Gregor', 'NNP'), ('s', 'NN')]\n",
    "\n",
    "# токены в которых притяжательное окончание присутствует tag_pos не умеет определять как NNP, то есть имена собственные\n",
    "print(nltk.pos_tag([\"Gregor's\"]))\n",
    "#[(\"Gregor's\", 'NN')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('troubled', 1),\n",
       " ('transformed', 1),\n",
       " ('vermin', 1),\n",
       " ('armour-like', 1),\n",
       " ('domed', 1),\n",
       " ('divided', 1),\n",
       " ('arches', 1),\n",
       " ('sections', 1),\n",
       " ('bedding', 1),\n",
       " ('cover', 1)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# выводим слова с сортировкой по возрастанию частот\n",
    "corpus.cfs(9,sort=1)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['your stomach sour',\n",
       " 'white fang hesitated',\n",
       " 'goo night everybody',\n",
       " 'hot words passed',\n",
       " 'howd it happen',\n",
       " 'give m time',\n",
       " 'something was happening',\n",
       " 'night had fallen',\n",
       " 'one eye watched',\n",
       " 'bring a light']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[12].keywords(rating=('rake', dict(max_words=3))).topn(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2415\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['And now was acknowledged the presence of the Red Death',\n",
       " 'Blood was its Avatar and its seal the redness and the horror of blood',\n",
       " 'It was in this apartment also that there stood against the western wall a gigantic clock of ebony',\n",
       " 'There were much of the beautiful much of the wanton much of the bizarre something of the terrible and not a little of that which might have excited disgust',\n",
       " 'To and fro in the seven chambers there stalked in fact a multitude of dreams',\n",
       " 'The figure was tall and gaunt and shrouded from head to foot in the habiliments of the grave',\n",
       " 'And then for a moment all is still and all is silent save the voice of the clock',\n",
       " 'But in spite of these things it was a gay and magnificent revel',\n",
       " 'It was in the blue room where stood the prince with a group of pale courtiers by his side',\n",
       " 'But these other apartments were densely crowded and in them beat feverishly the heart of life']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# много памяти для текстов имеющих больше 20 тыс. словоупотреблений... такой вот TextRank\n",
    "print(texts[6].nwords) # рассказ Эдгара По 'Маска Красной Смерти'\n",
    "texts[6].summarize(10, scores=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2119, 2119)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ndoc = 7\n",
    "words =  []\n",
    "for sent in texts[ndoc].sents():\n",
    "    words.extend(sent.lemmas())\n",
    "                     \n",
    "len(words),  texts[ndoc].count(words=False)                   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "def similarity(s1, s2):\n",
    "        '''Мера сходства - коэффициент Сёренсена - \n",
    "        https://ru.wikipedia.org/wiki/Коэффициент_Сёренсена\n",
    "        отношение количества одинаковых слов в \n",
    "        предложениях к суммарной длине предложений.\n",
    "        ''' \n",
    "        if not len(s1) or not len(s2):\n",
    "            return 0.0\n",
    "        \n",
    "        return len(s1.intersection(s2))/(1.0 * (len(s1) + len(s2)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "nsents = texts[ndoc].nsents    \n",
    "pairs = itertools.combinations(range(nsents), 2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11378564\n",
      "[(0, 1, 0.125), (0, 2, 0.15151515151515152), (0, 3, 0.125), (0, 4, 0.11538461538461539), (0, 5, 0.05555555555555555), (0, 6, 0.16666666666666666), (0, 7, 0.05263157894736842), (0, 8, 0.16), (0, 9, 0.08), (0, 10, 0.05)]\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for i, j in pairs:\n",
    "    sim = similarity(words[i], words[j])\n",
    "    if sim:\n",
    "        scores.append((i, j, sim)) \n",
    "    \n",
    "\n",
    "print(len(scores))\n",
    "print(scores[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['f', 'free']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].trie('free').prefixes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freely',\n",
       " 'freely',\n",
       " 'freely',\n",
       " 'freely']"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].trie('free').startswith"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freely',\n",
       " 'freely',\n",
       " 'freely',\n",
       " 'freely']"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].trie.startswith('free')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'free',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freedom',\n",
       " 'freely',\n",
       " 'freely',\n",
       " 'freely',\n",
       " 'freely']"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].trie('free').startswith"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1101, 37),\n",
       " (1535, 4),\n",
       " (1670, 7),\n",
       " (2907, 19),\n",
       " (3761, 26),\n",
       " (4587, 20),\n",
       " (4935, 21)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].trie.get('free', sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(\n",
       "\tname='doyle_the_adventures',\n",
       "\tencoding='ISO-8859-1',\n",
       "\tnsents=6794,\n",
       "\tnwords=104790,\n",
       "\tnlemmas=6639\n",
       ")"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'miss', 'missed', 'misses', 'missing', 'mission'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(texts[3].trie.startswith('miss'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TaggedSentence(\n",
       "\t'Man⁄NN⁄Man or⁄CC⁄or at⁄IN⁄at least⁄JJS⁄least criminal⁄JJ⁄criminal man⁄NN⁄man has⁄VBZ⁄have lost⁄VBN⁄lose all⁄DT⁄all enterprise⁄NN⁄enterprise and⁄CC⁄and originality⁄NN⁄originality',\n",
       "\t n=6195\n",
       ")"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].sents(6195)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(27, 11),\n",
       " (447, 19),\n",
       " (701, 37),\n",
       " (702, 21),\n",
       " (1051, 2),\n",
       " (1286, 13),\n",
       " (1566, 18),\n",
       " (1625, 5),\n",
       " (1625, 7),\n",
       " (1683, 6),\n",
       " (1832, 21),\n",
       " (2296, 14),\n",
       " (2585, 23),\n",
       " (2942, 12),\n",
       " (2952, 54),\n",
       " (3264, 13),\n",
       " (3265, 1),\n",
       " (3345, 39),\n",
       " (3347, 1),\n",
       " (3351, 26),\n",
       " (3447, 19),\n",
       " (3521, 9),\n",
       " (3796, 23),\n",
       " (4315, 14),\n",
       " (4388, 12),\n",
       " (5880, 24),\n",
       " (6180, 0),\n",
       " (6182, 10),\n",
       " (6188, 58),\n",
       " (6404, 24),\n",
       " (6407, 3),\n",
       " (6413, 57)]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occurrences = texts[3].trie['crime']\n",
    "occurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# список предложений в которые входи слово \"crime\"\n",
    "#[(nsent,texts[3].sents(nsent).raw) for nsent,_ in occurrences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(244, 0),\n",
       " (0, 4),\n",
       " (13, 2),\n",
       " (24, 5),\n",
       " (26, 31),\n",
       " (31, 36),\n",
       " (53, 2),\n",
       " (100, 7),\n",
       " (105, 1),\n",
       " (127, 0)]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# узнаем в каких предложениях и на каких позициях употреблялось данное слово\n",
    "texts[3].trie['holmes'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'holmes'"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].sents(24).words(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I had seen little of Holmes lately'"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].sents(24).raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The attention of the younger ones was then no longer to be gained by him'"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].sents(1103).raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#texts[1].trie['s']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "155595"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts[4].words(filtrate=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "155595"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[4].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "155595"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts[4].trie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6873"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts[0].words(uniq=True, lower=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6873"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].count(uniq=True, lower=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6387"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts[0].words(uniq=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6387"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].count(uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4958"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].count(words=False,uniq=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4958"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts[0].lemmas(uniq=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5947,\n",
       " 9709,\n",
       " 6787,\n",
       " 6794,\n",
       " 12526,\n",
       " 235,\n",
       " 101,\n",
       " 166,\n",
       " 3420,\n",
       " 780,\n",
       " 3409,\n",
       " 3207,\n",
       " 4755,\n",
       " 3730,\n",
       " 18262,\n",
       " 9694,\n",
       " 4898,\n",
       " 7341,\n",
       " 3850,\n",
       " 3287,\n",
       " 6504]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[texts[i].nsents for i in range(corpus.ndocs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'be': 4507, 'have': 2105, 'say': 603, 'do': 586, 'come': 340, 'see': 322, 'know': 267, 'go': 248, 'think': 238, 'take': 214, ...})"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "verbs = texts[3].postags(\"VERB\")\n",
    "verbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('be', 4507),\n",
       " ('have', 2105),\n",
       " ('say', 603),\n",
       " ('do', 586),\n",
       " ('come', 340),\n",
       " ('see', 322),\n",
       " ('know', 267),\n",
       " ('go', 248),\n",
       " ('think', 238),\n",
       " ('take', 214)]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].postags(\"VERB\", sort=-1)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TaggedSentence(\n",
       "\t'A⁄A⁄DT Scandal⁄Scandal⁄NNP in⁄in⁄IN Bohemia⁄Bohemia⁄NNP II⁄Ii⁄NNP',\n",
       "\t n=1\n",
       ")"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# получить ттегированное по частям речи представление прдложения\n",
    "texts[3].sents(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Token(word='a', idx=0, pos='DT', lemma='a', nsent=1),\n",
       " Token(word='scandal', idx=1, pos='NNP', lemma='scandal', nsent=1),\n",
       " Token(word='in', idx=2, pos='IN', lemma='in', nsent=1),\n",
       " Token(word='bohemia', idx=3, pos='NNP', lemma='bohemia', nsent=1),\n",
       " Token(word='ii', idx=4, pos='NNP', lemma='ii', nsent=1))"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].sents(1).tokens(lower=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['omne ignotum pro magnifico',\n",
       " 'prima donna imperial opera',\n",
       " 'botany variable geology profound',\n",
       " 'clark russell fine sea-stories',\n",
       " 'civilisation like untamed beasts',\n",
       " 'british barque sophy anderson',\n",
       " 'light mousseline de soie',\n",
       " 'mrs st clair assertion',\n",
       " 'san francisco cal u.s.a',\n",
       " 'settee said holmes relapsing']"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[3].keywords().topn(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def test(ndoc):\n",
    "    trie=sorted(texts[ndoc].trie().items(),key=lambda x:(x[1][0],x[1][1]))\n",
    "    from collections import defaultdict\n",
    "    sents = defaultdict(list)\n",
    "    for w,(nsent,idx) in trie:\n",
    "        tmp = idx\n",
    "        sents[nsent].append(w)\n",
    "        if tmp != idx:\n",
    "            #TaggedSentence(' '.join(sents[nsent])\n",
    "    return sents    \n",
    "                            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for ndoc in range(corpus.ndocs):\n",
    "#    %time x=test(ndoc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result = []\n",
    "#for ndocs in range(corpus.ndocs):\n",
    "#    result.append((ndocs,texts[ndocs].trie('s',sort=-1))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#for occur in result:\n",
    "#    if occur[1] != []:\n",
    "#        print(occur[0], texts[occur[0]].name)\n",
    "#        for sent in occur[1]:\n",
    "#            print(texts[occur[0]].sents(sent[0]).text, sent[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}