preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:02.507292Z",
     "start_time": "2020-04-25T12:25:01.816337Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:03.366139Z",
     "start_time": "2020-04-25T12:25:03.269405Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>School</th>\n",
       "      <th>Cohort</th>\n",
       "      <th>Book ID</th>\n",
       "      <th>Topic</th>\n",
       "      <th>Bookclub</th>\n",
       "      <th>User ID</th>\n",
       "      <th>Name</th>\n",
       "      <th>Message</th>\n",
       "      <th>Translation</th>\n",
       "      <th>Message Time</th>\n",
       "      <th>Is Answer</th>\n",
       "      <th>Page</th>\n",
       "      <th>Book relevance</th>\n",
       "      <th>Type</th>\n",
       "      <th>Category</th>\n",
       "      <th>CategoryBroad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OŠ Ketteja in Murna</td>\n",
       "      <td>MumaD</td>\n",
       "      <td>8</td>\n",
       "      <td>Kako bi lahko Cefizelj pobegnil policistu še n...</td>\n",
       "      <td>Book Club One</td>\n",
       "      <td>1382</td>\n",
       "      <td>MumaD8</td>\n",
       "      <td>gremo se pogovarjati</td>\n",
       "      <td>Let's talk</td>\n",
       "      <td>2019-06-18 05:16:16 AM</td>\n",
       "      <td>No</td>\n",
       "      <td>4</td>\n",
       "      <td>No</td>\n",
       "      <td>S</td>\n",
       "      <td>CE</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OŠ Ketteja in Murna</td>\n",
       "      <td>MumaD</td>\n",
       "      <td>8</td>\n",
       "      <td>Kako bi lahko Cefizelj pobegnil policistu še n...</td>\n",
       "      <td>Book Club One</td>\n",
       "      <td>1392</td>\n",
       "      <td>MumaD18</td>\n",
       "      <td>Kip je to</td>\n",
       "      <td>This is a statue</td>\n",
       "      <td>2019-06-18 05:17:29 AM</td>\n",
       "      <td>No</td>\n",
       "      <td>4</td>\n",
       "      <td>No</td>\n",
       "      <td>S</td>\n",
       "      <td>CO</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>OŠ Ketteja in Murna</td>\n",
       "      <td>MumaD</td>\n",
       "      <td>8</td>\n",
       "      <td>Kako bi lahko Cefizelj pobegnil policistu še n...</td>\n",
       "      <td>Book Club One</td>\n",
       "      <td>1392</td>\n",
       "      <td>MumaD18</td>\n",
       "      <td>Kdo je to jaz sem tara</td>\n",
       "      <td>Who is this I am Tara (girl's name)</td>\n",
       "      <td>2019-06-18 05:17:59 AM</td>\n",
       "      <td>No</td>\n",
       "      <td>4</td>\n",
       "      <td>No</td>\n",
       "      <td>Q</td>\n",
       "      <td>IQ</td>\n",
       "      <td>I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OŠ Ketteja in Murna</td>\n",
       "      <td>MumaD</td>\n",
       "      <td>8</td>\n",
       "      <td>Kako bi lahko Cefizelj pobegnil policistu še n...</td>\n",
       "      <td>Book Club One</td>\n",
       "      <td>1382</td>\n",
       "      <td>MumaD8</td>\n",
       "      <td>kaj kip</td>\n",
       "      <td>what statue</td>\n",
       "      <td>2019-06-18 05:18:58 AM</td>\n",
       "      <td>No</td>\n",
       "      <td>4</td>\n",
       "      <td>No</td>\n",
       "      <td>S</td>\n",
       "      <td>CO</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OŠ Ketteja in Murna</td>\n",
       "      <td>MumaD</td>\n",
       "      <td>8</td>\n",
       "      <td>Kako bi lahko Cefizelj pobegnil policistu še n...</td>\n",
       "      <td>Book Club One</td>\n",
       "      <td>1382</td>\n",
       "      <td>MumaD8</td>\n",
       "      <td>gremo ven</td>\n",
       "      <td>let's go outside</td>\n",
       "      <td>2019-06-18 05:19:24 AM</td>\n",
       "      <td>No</td>\n",
       "      <td>4</td>\n",
       "      <td>No</td>\n",
       "      <td>S</td>\n",
       "      <td>CO</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>OŠ Koseze</td>\n",
       "      <td>Koseze</td>\n",
       "      <td>2</td>\n",
       "      <td>Mojca je bila prepričana, da je z njo ženska i...</td>\n",
       "      <td>Knjižni Klub 8</td>\n",
       "      <td>676</td>\n",
       "      <td>karin japelj K</td>\n",
       "      <td>Jaz mislim,da ne,ker je Pehta zlobna.</td>\n",
       "      <td>I don't think so, because Pehta is mena.</td>\n",
       "      <td>12.3.2019 5:51</td>\n",
       "      <td>No</td>\n",
       "      <td>5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A</td>\n",
       "      <td>DA</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>OŠ Koseze</td>\n",
       "      <td>Koseze</td>\n",
       "      <td>2</td>\n",
       "      <td>Mojca je bila prepričana, da je z njo ženska i...</td>\n",
       "      <td>Knjižni Klub 8</td>\n",
       "      <td>726</td>\n",
       "      <td>timskander</td>\n",
       "      <td>Jaz mislim da ne, ker je zlobna.</td>\n",
       "      <td>I don't think so, because she is mean.</td>\n",
       "      <td>12.3.2019 5:53</td>\n",
       "      <td>No</td>\n",
       "      <td>14</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A</td>\n",
       "      <td>DA</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3263</th>\n",
       "      <td>OŠ Koseze</td>\n",
       "      <td>Koseze</td>\n",
       "      <td>2</td>\n",
       "      <td>Mojca je bila prepričana, da je z njo ženska i...</td>\n",
       "      <td>Knjižni Klub 8</td>\n",
       "      <td>676</td>\n",
       "      <td>karin japelj K</td>\n",
       "      <td>itak</td>\n",
       "      <td>Of course.</td>\n",
       "      <td>12.3.2019 6:24</td>\n",
       "      <td>No</td>\n",
       "      <td>14</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A</td>\n",
       "      <td>DA</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3264</th>\n",
       "      <td>OŠ Koseze</td>\n",
       "      <td>Koseze</td>\n",
       "      <td>2</td>\n",
       "      <td>Mojca je bila prepričana, da je z njo ženska i...</td>\n",
       "      <td>Knjižni Klub 8</td>\n",
       "      <td>676</td>\n",
       "      <td>karin japelj K</td>\n",
       "      <td>oops</td>\n",
       "      <td>oops</td>\n",
       "      <td>12.3.2019 6:25</td>\n",
       "      <td>No</td>\n",
       "      <td>14</td>\n",
       "      <td>No</td>\n",
       "      <td>S</td>\n",
       "      <td>CO</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3265</th>\n",
       "      <td>OŠ Koseze</td>\n",
       "      <td>Koseze</td>\n",
       "      <td>2</td>\n",
       "      <td>Mojca je bila prepričana, da je z njo ženska i...</td>\n",
       "      <td>Knjižni Klub 8</td>\n",
       "      <td>676</td>\n",
       "      <td>karin japelj K</td>\n",
       "      <td>ja</td>\n",
       "      <td>Yes</td>\n",
       "      <td>12.3.2019 6:26</td>\n",
       "      <td>No</td>\n",
       "      <td>15</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A</td>\n",
       "      <td>DA</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3266 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   School  Cohort  Book ID  \\\n",
       "0     OŠ Ketteja in Murna   MumaD        8   \n",
       "1     OŠ Ketteja in Murna   MumaD        8   \n",
       "2     OŠ Ketteja in Murna   MumaD        8   \n",
       "3     OŠ Ketteja in Murna   MumaD        8   \n",
       "4     OŠ Ketteja in Murna   MumaD        8   \n",
       "...                   ...     ...      ...   \n",
       "3261            OŠ Koseze  Koseze        2   \n",
       "3262            OŠ Koseze  Koseze        2   \n",
       "3263            OŠ Koseze  Koseze        2   \n",
       "3264            OŠ Koseze  Koseze        2   \n",
       "3265            OŠ Koseze  Koseze        2   \n",
       "\n",
       "                                                  Topic        Bookclub  \\\n",
       "0     Kako bi lahko Cefizelj pobegnil policistu še n...   Book Club One   \n",
       "1     Kako bi lahko Cefizelj pobegnil policistu še n...   Book Club One   \n",
       "2     Kako bi lahko Cefizelj pobegnil policistu še n...   Book Club One   \n",
       "3     Kako bi lahko Cefizelj pobegnil policistu še n...   Book Club One   \n",
       "4     Kako bi lahko Cefizelj pobegnil policistu še n...   Book Club One   \n",
       "...                                                 ...             ...   \n",
       "3261  Mojca je bila prepričana, da je z njo ženska i...  Knjižni Klub 8   \n",
       "3262  Mojca je bila prepričana, da je z njo ženska i...  Knjižni Klub 8   \n",
       "3263  Mojca je bila prepričana, da je z njo ženska i...  Knjižni Klub 8   \n",
       "3264  Mojca je bila prepričana, da je z njo ženska i...  Knjižni Klub 8   \n",
       "3265  Mojca je bila prepričana, da je z njo ženska i...  Knjižni Klub 8   \n",
       "\n",
       "      User ID            Name                                Message  \\\n",
       "0        1382          MumaD8                   gremo se pogovarjati   \n",
       "1        1392         MumaD18                              Kip je to   \n",
       "2        1392         MumaD18                 Kdo je to jaz sem tara   \n",
       "3        1382          MumaD8                                kaj kip   \n",
       "4        1382          MumaD8                              gremo ven   \n",
       "...       ...             ...                                    ...   \n",
       "3261      676  karin japelj K  Jaz mislim,da ne,ker je Pehta zlobna.   \n",
       "3262      726      timskander       Jaz mislim da ne, ker je zlobna.   \n",
       "3263      676  karin japelj K                                   itak   \n",
       "3264      676  karin japelj K                                   oops   \n",
       "3265      676  karin japelj K                                     ja   \n",
       "\n",
       "                                 Translation              Message Time  \\\n",
       "0                                   Let's talk  2019-06-18 05:16:16 AM   \n",
       "1                             This is a statue  2019-06-18 05:17:29 AM   \n",
       "2          Who is this I am Tara (girl's name)  2019-06-18 05:17:59 AM   \n",
       "3                                  what statue  2019-06-18 05:18:58 AM   \n",
       "4                             let's go outside  2019-06-18 05:19:24 AM   \n",
       "...                                        ...                     ...   \n",
       "3261  I don't think so, because Pehta is mena.          12.3.2019 5:51   \n",
       "3262   I don't think so, because she is mean.           12.3.2019 5:53   \n",
       "3263                               Of course.           12.3.2019 6:24   \n",
       "3264                                      oops          12.3.2019 6:25   \n",
       "3265                                       Yes          12.3.2019 6:26   \n",
       "\n",
       "     Is Answer  Page Book relevance Type Category CategoryBroad  \n",
       "0           No     4             No    S       CE             C  \n",
       "1           No     4             No    S       CO             C  \n",
       "2           No     4             No    Q       IQ             I  \n",
       "3           No     4             No    S       CO             C  \n",
       "4           No     4             No    S       CO             C  \n",
       "...        ...   ...            ...  ...      ...           ...  \n",
       "3261        No     5            Yes    A       DA             D  \n",
       "3262        No    14            Yes    A       DA             D  \n",
       "3263        No    14            Yes    A       DA             D  \n",
       "3264        No    14             No    S       CO             C  \n",
       "3265        No    15            Yes    A       DA             D  \n",
       "\n",
       "[3266 rows x 16 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read data\n",
    "dataset_path = 'data/discussion_data.csv'\n",
    "df = pd.read_csv(dataset_path, delimiter=';')\n",
    "df['CategoryBroad'] = pd.Categorical(df['CategoryBroad'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['OŠ Ketteja in Murna', 'OŠ Franca Rozmana Staneta',\n",
       "       'OŠ Nove Fužine', 'OŠ Alojzija Šuštarja', 'OŠ Vižmarje - Brod',\n",
       "       'OŠ Vide Pregarc', 'OŠ Valentina Vodnika', 'OŠ Koseze'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.School.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:03.903019Z",
     "start_time": "2020-04-25T12:25:03.896646Z"
    }
   },
   "outputs": [],
   "source": [
    "df.Message = df.Message.fillna('')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Message parsing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:05.397749Z",
     "start_time": "2020-04-25T12:25:05.040236Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "from nltk import word_tokenize\n",
    "from nltk.tokenize.casual import TweetTokenizer\n",
    "from nltk.tokenize.destructive import NLTKWordTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:05.583051Z",
     "start_time": "2020-04-25T12:25:05.572761Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = TweetTokenizer(preserve_case=False, reduce_len=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:06.045143Z",
     "start_time": "2020-04-25T12:25:06.033496Z"
    }
   },
   "outputs": [],
   "source": [
    "# Split punctuation without space ('neki.halo' -> 'neki', '.', 'halo')\n",
    "def split_punc(tokens):\n",
    "    tokens_out = []\n",
    "    for token in tokens:\n",
    "        match = re.match('(\\w+)\\.(\\w+)', token)\n",
    "        if match is not None:\n",
    "            l,r = match.groups()\n",
    "            tokens_out.extend([l, '.', r])\n",
    "        else:\n",
    "            tokens_out.append(token)\n",
    "            \n",
    "    return tokens_out\n",
    "\n",
    "# Split numbers to separate tokens ('username15halo' -> 'username', '15', 'halo')\n",
    "def split_num(tokens):\n",
    "    tokens_out = []\n",
    "    for token in tokens:\n",
    "        new_tokens = re.findall('\\d+|\\D+', token)\n",
    "        tokens_out.extend(new_tokens)\n",
    "            \n",
    "    return tokens_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:06.692383Z",
     "start_time": "2020-04-25T12:25:06.686666Z"
    }
   },
   "outputs": [],
   "source": [
    "def tokenize(message):\n",
    "    tokens = tokenizer.tokenize(message)\n",
    "    \n",
    "    # Fix punctuations\n",
    "    tokens = split_punc(tokens)\n",
    "    \n",
    "    # Separate numbers\n",
    "    tokens = split_num(tokens)\n",
    "    \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:03:04.929142Z",
     "start_time": "2020-04-25T16:03:04.917286Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40                                                nevem\n",
       "41                                           Kaj ne ves\n",
       "42                              A splotches ves kdo sem\n",
       "43    <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<...\n",
       "44                                                 Nehi\n",
       "45    llllllllllllllllllllllllllllllllllllllllllllll...\n",
       "46                                                   ??\n",
       "47                                         tara neni ti\n",
       "48                                            Ne napacn\n",
       "49                                                  vem\n",
       "50                                          kaj mislite\n",
       "51                                                nevem\n",
       "52                                             odgovori\n",
       "53                             ker je tako boljzanimivo\n",
       "54                                     kaj mislis s tem\n",
       "55                                               ker je\n",
       "56                                       Kako to mislis\n",
       "57                                            kdo si ti\n",
       "58                                               Do jst\n",
       "59                                 natalia vem da si ti\n",
       "Name: Message, dtype: object"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages = df.Message\n",
    "messages.iloc[40:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:16.686176Z",
     "start_time": "2020-04-25T12:25:16.678782Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ups', '.', 'mislila', 'sem', 'kdo', 'jst', '.']\n"
     ]
    }
   ],
   "source": [
    "i=67\n",
    "print(tokenize(messages[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:21.500638Z",
     "start_time": "2020-04-25T12:25:21.216508Z"
    }
   },
   "outputs": [],
   "source": [
    "conversations = [tokenize(message) for message in messages]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:02:55.511158Z",
     "start_time": "2020-04-25T16:02:55.497132Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['nev'],\n",
       " ['kaj'],\n",
       " ['splotches', 'kdo'],\n",
       " ['<', '<', '<'],\n",
       " ['neh'],\n",
       " ['lllooojjje', '<', '<', '<'],\n",
       " ['?', '?'],\n",
       " ['tar', 'nen'],\n",
       " ['napacn'],\n",
       " ['vedeti'],\n",
       " ['kaj', 'misliti'],\n",
       " ['nev'],\n",
       " ['odgovor'],\n",
       " ['boljzanimiv'],\n",
       " ['kaj', 'mislisa', 'ta'],\n",
       " [],\n",
       " ['kak', 'mislisa'],\n",
       " ['kdo'],\n",
       " ['jst'],\n",
       " ['natalio', 'vedeti']]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversations[40:60]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop words removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:24.748500Z",
     "start_time": "2020-04-25T12:25:24.741052Z"
    }
   },
   "outputs": [],
   "source": [
    "class StopWordsRemover():\n",
    "    def __init__(self, stop_words_txt):\n",
    "        with open(stop_words_txt, 'r') as file:\n",
    "            self.stopwords = {line.strip() for line in file}\n",
    "            \n",
    "    def remove_stopwords(self, tokens):\n",
    "        filtered = [token for token in tokens if token not in self.stopwords]\n",
    "        return filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:25.017197Z",
     "start_time": "2020-04-25T12:25:25.005858Z"
    }
   },
   "outputs": [],
   "source": [
    "stopwords_remover = StopWordsRemover('data/stopwords-sl-custom.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:25.384493Z",
     "start_time": "2020-04-25T12:25:25.379881Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['zakaj', 'to', 'pises'], ['zakaj', 'pises'])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = conversations[21]\n",
    "tokens, stopwords_remover.remove_stopwords(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:25.618221Z",
     "start_time": "2020-04-25T12:25:25.610530Z"
    }
   },
   "outputs": [],
   "source": [
    "conversations = [stopwords_remover.remove_stopwords(tokens) for tokens in conversations]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:26.261704Z",
     "start_time": "2020-04-25T12:25:26.248292Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['poklicati', 'telefonu']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversations[1010]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lematizacija"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:28.245999Z",
     "start_time": "2020-04-25T12:25:28.224261Z"
    }
   },
   "outputs": [],
   "source": [
    "import lemmagen.lemmatizer\n",
    "from lemmagen.lemmatizer import Lemmatizer\n",
    "\n",
    "lemmatizer = Lemmatizer(dictionary=lemmagen.DICTIONARY_SLOVENE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:28.578629Z",
     "start_time": "2020-04-25T12:25:28.552319Z"
    }
   },
   "outputs": [],
   "source": [
    "conversations = [[lemmatizer.lemmatize(token) for token in message] for message in conversations]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:29.054489Z",
     "start_time": "2020-04-25T12:25:29.037070Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['iti', 'pogovarjati'],\n",
       " ['kip'],\n",
       " ['kdo', 'tar'],\n",
       " ['kaj', 'kip'],\n",
       " ['iti', 'vena'],\n",
       " ['kam'],\n",
       " ['park'],\n",
       " ['zadusiti', 'drug'],\n",
       " ['park', 'kodeljev'],\n",
       " ['kdo', 'pis', 'moči'],\n",
       " ['gregr'],\n",
       " ['gregor'],\n",
       " ['behinda'],\n",
       " ['neh'],\n",
       " ['bols', 'poves', 'kdo'],\n",
       " ['tar'],\n",
       " ['ooonnnee'],\n",
       " ['ok'],\n",
       " ['zzzaaakkkaaajjje'],\n",
       " ['reklo', 'pogledati', 'ta', 'zbezati'],\n",
       " ['vvvtttfff'],\n",
       " ['zakaj', 'pises'],\n",
       " ['kdo', 'mumada', '24'],\n",
       " ['ubiti'],\n",
       " ['magoca', 'postati', 'prevec', 'nasilen', '?', '?'],\n",
       " ['ja', 'nasilen'],\n",
       " ['napa', 'is', 'kaj'],\n",
       " ['kaj'],\n",
       " ['upati', 'nov', 'naslednji', 'kniga', 'stup', 'cav'],\n",
       " ['hus', 'uijkh'],\n",
       " [],\n",
       " ['gregor'],\n",
       " ['ohč',\n",
       "  '3',\n",
       "  '.',\n",
       "  'hizti',\n",
       "  '.',\n",
       "  'zzz',\n",
       "  'wehrw',\n",
       "  'ebzztdftzdztdtziztdti',\n",
       "  '45',\n",
       "  'wweddfdfffdfbfbdrez',\n",
       "  '5',\n",
       "  'z',\n",
       "  'rt',\n",
       "  'fg',\n",
       "  'rtrthfhgf',\n",
       "  'fthfhfthfthfthfthfthfhfghfthzfrtje',\n",
       "  'gz---aw',\n",
       "  'wawdwawsdawd',\n",
       "  'trrreo',\n",
       "  'daws',\n",
       "  'aadw',\n",
       "  'a',\n",
       "  'wdawsdasdssawww',\n",
       "  'fff',\n",
       "  'w',\n",
       "  'awdadawdwdawdawdadjadvjawgdagdhasxbasjkhdkawbkjasgdawghdkaghdkajkgawkgd',\n",
       "  'kagfjhgsfvawgdadjzgfinjzjgujlizuukgguyd',\n",
       "  'jgdt',\n",
       "  ',',\n",
       "  'ttv',\n",
       "  '76',\n",
       "  'uzfrhtthdhjdftdtttzezgzuutttzzzrdtdzka',\n",
       "  'ktfgca',\n",
       "  'iluehz',\n",
       "  'weu',\n",
       "  'hsehfiwhf',\n",
       "  'usrhfsiufhweufhwekhsehfkshuefksuf',\n",
       "  'kufhskuhzezhshkeufszehwehzefhs',\n",
       "  'fhwefefsefsefsfesfeww',\n",
       "  'sssohgkgghuočg',\n",
       "  'utilo'],\n",
       " ['ja'],\n",
       " ['neh'],\n",
       " ['gn',\n",
       "  'sekhsoiefheiohfioehfehfrehkflwefhlief',\n",
       "  'erilčeilzwhreibaeizcsfhsezfzscjsgweuwtrewfasegwegerrrsjhfesgvfkshefjiseifhsuieffnčiejkfsaočejkfw-čkfwefjoisejliw',\n",
       "  ','],\n",
       " ['asje',\n",
       "  ',',\n",
       "  'dbfsfuoisurg',\n",
       "  '4',\n",
       "  'iwr',\n",
       "  '0',\n",
       "  'faehfgydlfhkskdfjskjgnsdvfnysjvfnsdka',\n",
       "  '.',\n",
       "  'eglsirhulghsbfjsdr'],\n",
       " ['zakaj', 'pogovor'],\n",
       " ['fortniti'],\n",
       " ['nev'],\n",
       " ['nev'],\n",
       " ['kaj'],\n",
       " ['splotches', 'kdo'],\n",
       " ['<', '<', '<'],\n",
       " ['neh'],\n",
       " ['lllooojjje', '<', '<', '<'],\n",
       " ['?', '?'],\n",
       " ['tar', 'nen'],\n",
       " ['napacn'],\n",
       " ['vedeti'],\n",
       " ['kaj', 'misliti'],\n",
       " ['nev'],\n",
       " ['odgovor'],\n",
       " ['boljzanimiv'],\n",
       " ['kaj', 'mislisa', 'ta'],\n",
       " [],\n",
       " ['kak', 'mislisa'],\n",
       " ['kdo'],\n",
       " ['jst'],\n",
       " ['natalio', 'vedeti'],\n",
       " ['pisateljica', 'met', 'domisliti', '?', '?'],\n",
       " ['kdo'],\n",
       " ['.'],\n",
       " ['kaj', 'pomen', 'dst', 'nesti'],\n",
       " ['kdo'],\n",
       " ['ja'],\n",
       " ['tar'],\n",
       " ['ups', '.', 'misliti', 'kdo', 'jst', '.'],\n",
       " ['vtf'],\n",
       " ['kaj'],\n",
       " ['jurje', 'pis', 'kletvica'],\n",
       " ['natalija'],\n",
       " ['kaj', 'natalija'],\n",
       " ['grega'],\n",
       " ['nev'],\n",
       " ['kaj', 'nevo'],\n",
       " [],\n",
       " ['kdo', 'mum', '24'],\n",
       " ['kdo'],\n",
       " ['tar'],\n",
       " ['gregor'],\n",
       " ['natalija'],\n",
       " ['midlifa', 'imaga', 'planet', 'custvo', '.'],\n",
       " ['ta', 'hotel', 'pokazati', 'prijateljstvo', '.', '?', '?'],\n",
       " [],\n",
       " ['kaksen', 'odgovor'],\n",
       " ['usec', 'knjiga'],\n",
       " ['smiseln', '.'],\n",
       " ['ja', '.'],\n",
       " ['kdo'],\n",
       " ['kdo', 'mumada', '4'],\n",
       " ['natalio'],\n",
       " ['mio', '.'],\n",
       " ['povedati'],\n",
       " ['kaj', 'pš', 'zgodba'],\n",
       " ['odgovoriti'],\n",
       " ['kakati'],\n",
       " ['za'],\n",
       " ['dgfvhvdccgflyegyvuvba'],\n",
       " ['nev']]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversations[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dešumifikacija"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:31.469357Z",
     "start_time": "2020-04-25T12:25:31.450580Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "class RoofRemoval():\n",
    "    def __init__(self):\n",
    "        self.replacements = dict([('ž', 'z'),('č', 'c'),('š', 's'),('ć', 'c'),('đ', 'dz')])\n",
    "        self.pattern = re.compile(\"|\".join(self.replacements.keys()))\n",
    "\n",
    "    def remove(self, token):\n",
    "        token = self.pattern.sub(lambda x: self.replacements[re.escape(x.group(0))], token)\n",
    "        return token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:31.835552Z",
     "start_time": "2020-04-25T12:25:31.829642Z"
    }
   },
   "outputs": [],
   "source": [
    "roof_removal = RoofRemoval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:32.252126Z",
     "start_time": "2020-04-25T12:25:32.234866Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'test sumnikov. da vidimo ce pobere csz?'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roof_removal.remove('test šumnikov. da vidimo če pobere čšž?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:25:34.333294Z",
     "start_time": "2020-04-25T12:25:34.313521Z"
    }
   },
   "outputs": [],
   "source": [
    "conversations = [[roof_removal.remove(token) for token in message] for message in conversations]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Error correction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T10:40:19.267262Z",
     "start_time": "2020-04-25T10:40:19.250042Z"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.metrics.distance import edit_distance\n",
    "import difflib\n",
    "\n",
    "class SpellingCorrection():\n",
    "    def __init__(self, dict_txt, roof_removal=None):\n",
    "        \n",
    "        with open(dict_txt, 'r') as file:\n",
    "            dictionary = (line.strip() for line in file)\n",
    "            \n",
    "            if roof_removal is not None:\n",
    "                dictionary = (roof_removal.remove(w) for w in dictionary)\n",
    "                \n",
    "            self.dictionary = list(dictionary)\n",
    "            \n",
    "        self.roof_removal = roof_removal\n",
    "            \n",
    "    def find_close(self, token, n=1, cutoff=0.9):\n",
    "        close_matches = difflib.get_close_matches(token, self.dictionary, n=n, cutoff=cutoff)\n",
    "        \n",
    "        scores = [difflib.SequenceMatcher(None, token, w).ratio() for w in close_matches]\n",
    "        \n",
    "        return list(zip(close_matches, scores))\n",
    "    \n",
    "    def replace_if_close(self, token, thresh=0.9):\n",
    "        res = self.find_close(token, n=1, cutoff=thresh)\n",
    "        if len(res) < 1:\n",
    "            return token\n",
    "        \n",
    "        word, sim = res[0]\n",
    "        if sim < 1:\n",
    "            print(f'{token} -> {word} ({sim})')\n",
    "        \n",
    "        return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T10:40:27.603677Z",
     "start_time": "2020-04-25T10:40:26.991982Z"
    }
   },
   "outputs": [],
   "source": [
    "spelling_correction = SpellingCorrection('data/dict-sl.txt', roof_removal=roof_removal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T10:40:29.412787Z",
     "start_time": "2020-04-25T10:40:28.678357Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avtobrus -> avtobus (0.9333333333333333)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'avtobus'"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spelling_correction.replace_if_close('avtobrus')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-24T21:17:07.181770Z",
     "start_time": "2020-04-24T21:17:07.176653Z"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-24T22:28:53.817317Z",
     "start_time": "2020-04-24T21:17:09.460115Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bbe8a236f69418882171b8440f28655",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gregr -> gregor (0.9090909090909091)\n",
      "pises -> prises (0.9090909090909091)\n",
      "kniga -> knjiga (0.9090909090909091)\n",
      "utilo -> cutilo (0.9090909090909091)\n",
      "napacn -> napacen (0.9230769230769231)\n",
      "jurje -> jurjev (0.9090909090909091)\n",
      "jurje -> jurjev (0.9090909090909091)\n",
      "napisan -> nadpisan (0.9333333333333333)\n",
      "plaziti -> vplaziti (0.9333333333333333)\n",
      "jurje -> jurjev (0.9090909090909091)\n",
      "jurje -> jurjev (0.9090909090909091)\n",
      "odgovpor -> odgovor (0.9333333333333333)\n",
      "obcutka -> obcutkar (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "adrian -> arian (0.9090909090909091)\n",
      "natas -> naftas (0.9090909090909091)\n",
      "primeti -> primreti (0.9333333333333333)\n",
      "primeti -> primreti (0.9333333333333333)\n",
      "zacniti -> zlacniti (0.9333333333333333)\n",
      "zacniti -> zlacniti (0.9333333333333333)\n",
      "volitve -> volitven (0.9333333333333333)\n",
      "nasljeden -> nasleden (0.9411764705882353)\n",
      "nasljednji -> naslednji (0.9473684210526315)\n",
      "navciti -> naviti (0.9230769230769231)\n",
      "skocit -> skociti (0.9230769230769231)\n",
      "nobeden -> obeden (0.9230769230769231)\n",
      "mogoca -> mogoc (0.9090909090909091)\n",
      "truditi -> utruditi (0.9333333333333333)\n",
      "najboljs -> najboljsi (0.9411764705882353)\n",
      "perdeti -> prdeti (0.9230769230769231)\n",
      "pocel -> polcel (0.9090909090909091)\n",
      "klosar -> kosar (0.9090909090909091)\n",
      "klosar -> kosar (0.9090909090909091)\n",
      "alisati -> zalisati (0.9333333333333333)\n",
      "alisati -> zalisati (0.9333333333333333)\n",
      "kakatiz -> kakati (0.9230769230769231)\n",
      "debiti -> debit (0.9090909090909091)\n",
      "lahkon -> lahon (0.9090909090909091)\n",
      "drkat -> drkati (0.9090909090909091)\n",
      "tajan -> tajgan (0.9090909090909091)\n",
      "klosar -> kosar (0.9090909090909091)\n",
      "tajan -> tajgan (0.9090909090909091)\n",
      "kereten -> kreten (0.9230769230769231)\n",
      "spreme -> supreme (0.9230769230769231)\n",
      "digen -> diogen (0.9090909090909091)\n",
      "bolje -> boljek (0.9090909090909091)\n",
      "zdejati -> zdrejati (0.9333333333333333)\n",
      "hitler -> hiter (0.9090909090909091)\n",
      "mihael -> mihel (0.9090909090909091)\n",
      "hitler -> hiter (0.9090909090909091)\n",
      "mihael -> mihel (0.9090909090909091)\n",
      "mihael -> mihel (0.9090909090909091)\n",
      "mihael -> mihel (0.9090909090909091)\n",
      "hitler -> hiter (0.9090909090909091)\n",
      "hitler -> hiter (0.9090909090909091)\n",
      "zalublen -> zaluben (0.9333333333333333)\n",
      "utapjati -> utapljati (0.9411764705882353)\n",
      "umera -> numera (0.9090909090909091)\n",
      "negda -> negoda (0.9090909090909091)\n",
      "npisati -> napisati (0.9333333333333333)\n",
      "klosara -> klosarka (0.9333333333333333)\n",
      "debiti -> debit (0.9090909090909091)\n",
      "klosara -> klosarka (0.9333333333333333)\n",
      "klosara -> klosarka (0.9333333333333333)\n",
      "klosariti -> kosariti (0.9411764705882353)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "zdejati -> zdrejati (0.9333333333333333)\n",
      "daniti -> dahniti (0.9230769230769231)\n",
      "pogvarjati -> pogovarjati (0.9523809523809523)\n",
      "zdejati -> zdrejati (0.9333333333333333)\n",
      "daniti -> dahniti (0.9230769230769231)\n",
      "zamejaz -> zameja (0.9230769230769231)\n",
      "zdejati -> zdrejati (0.9333333333333333)\n",
      "vidti -> videti (0.9090909090909091)\n",
      "uprasanje -> prasanje (0.9411764705882353)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "negres -> neres (0.9090909090909091)\n",
      "ljudjee -> ljudje (0.9230769230769231)\n",
      "sprasevati -> prasevati (0.9473684210526315)\n",
      "pmagati -> pomagati (0.9333333333333333)\n",
      "kristina -> krstina (0.9333333333333333)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "obrnjen -> obranjen (0.9333333333333333)\n",
      "uprasati -> uprascati (0.9411764705882353)\n",
      "zeloda -> zelodar (0.9230769230769231)\n",
      "pocel -> polcel (0.9090909090909091)\n",
      "goricek -> gricek (0.9230769230769231)\n",
      "opero -> operoz (0.9090909090909091)\n",
      "fiaska -> faska (0.9090909090909091)\n",
      "zeloda -> zelodar (0.9230769230769231)\n",
      "kirilo -> krilo (0.9090909090909091)\n",
      "bratca -> braca (0.9090909090909091)\n",
      "tacek -> tracek (0.9090909090909091)\n",
      "mednariden -> mednaroden (0.9)\n",
      "zodba -> zgodba (0.9090909090909091)\n",
      "rabla -> rambla (0.9090909090909091)\n",
      "razmislati -> razmisljati (0.9523809523809523)\n",
      "nauca -> naucba (0.9090909090909091)\n",
      "ponavljassti -> ponavljati (0.9090909090909091)\n",
      "bolti -> boliti (0.9090909090909091)\n",
      "bolsa -> bolsak (0.9090909090909091)\n",
      "golfa -> golfar (0.9090909090909091)\n",
      "pocas -> polcas (0.9090909090909091)\n",
      "zaustajati -> zastajati (0.9473684210526315)\n",
      "zdejati -> zdrejati (0.9333333333333333)\n",
      "posla -> poslad (0.9090909090909091)\n",
      "pises -> prises (0.9090909090909091)\n",
      "lenje -> tlenje (0.9090909090909091)\n",
      "lenje -> tlenje (0.9090909090909091)\n",
      "hahah -> hahaha (0.9090909090909091)\n",
      "storgati -> strgati (0.9333333333333333)\n",
      "zanimiov -> zanimiv (0.9333333333333333)\n",
      "mogoca -> mogoc (0.9090909090909091)\n",
      "razloza -> razlozba (0.9333333333333333)\n",
      "vrsti -> vrsiti (0.9090909090909091)\n",
      "epovedati -> povedati (0.9411764705882353)\n",
      "nasmucarski -> nesmucarski (0.9090909090909091)\n",
      "spremlati -> spremljati (0.9473684210526315)\n",
      "igricaa -> igrica (0.9230769230769231)\n",
      "marel -> martel (0.9090909090909091)\n",
      "govort -> govor (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "sprehajati -> prehajati (0.9473684210526315)\n",
      "strmogljavljkati -> strmoglavljati (0.9333333333333333)\n",
      "strmogljavljati -> strmoglavljati (0.9655172413793104)\n",
      "boljs -> boljsi (0.9090909090909091)\n",
      "zdejati -> zdrejati (0.9333333333333333)\n",
      "kitajcka -> kitajka (0.9333333333333333)\n",
      "pametn -> pameton (0.9230769230769231)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "pametn -> pameton (0.9230769230769231)\n",
      "zdejati -> zdrejati (0.9333333333333333)\n",
      "seved -> vseved (0.9090909090909091)\n",
      "lenje -> tlenje (0.9090909090909091)\n",
      "franca -> francka (0.9230769230769231)\n",
      "lenje -> tlenje (0.9090909090909091)\n",
      "zasel -> zrasel (0.9090909090909091)\n",
      "rabti -> rabiti (0.9090909090909091)\n",
      "zasel -> zrasel (0.9090909090909091)\n",
      "lenje -> tlenje (0.9090909090909091)\n",
      "plisa -> pliska (0.9090909090909091)\n",
      "copika -> copka (0.9090909090909091)\n",
      "francija -> francijas (0.9411764705882353)\n",
      "nerazumeti -> nerazumet (0.9473684210526315)\n",
      "avtomacski -> avtomatski (0.9)\n",
      "pripevovedovati -> pripovedovati (0.9285714285714286)\n",
      "nepovedati -> napovedati (0.9)\n",
      "spreminjati -> preminjati (0.9523809523809523)\n",
      "piskota -> piskotav (0.9333333333333333)\n",
      "mogoca -> mogoc (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "izgledati -> zgledati (0.9411764705882353)\n",
      "hahah -> hahaha (0.9090909090909091)\n",
      "emiti -> zemiti (0.9090909090909091)\n",
      "proilen -> prozilen (0.9333333333333333)\n",
      "emiti -> zemiti (0.9090909090909091)\n",
      "kirati -> ksirati (0.9230769230769231)\n",
      "david -> dravid (0.9090909090909091)\n",
      "emiti -> zemiti (0.9090909090909091)\n",
      "emiti -> zemiti (0.9090909090909091)\n",
      "domislija -> domisljija (0.9473684210526315)\n",
      "poskusati -> pokusati (0.9411764705882353)\n",
      "brigati -> zbrigati (0.9333333333333333)\n",
      "pollicaj -> policaj (0.9333333333333333)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "brigati -> zbrigati (0.9333333333333333)\n",
      "zamieti -> zamiti (0.9230769230769231)\n",
      "movedati -> movdati (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "mogoca -> mogoc (0.9090909090909091)\n",
      "porosta -> porota (0.9230769230769231)\n",
      "david -> dravid (0.9090909090909091)\n",
      "nauca -> naucba (0.9090909090909091)\n",
      "beati -> bezati (0.9090909090909091)\n",
      "posati -> postati (0.9230769230769231)\n",
      "nepovedati -> napovedati (0.9)\n",
      "veverickati -> vevericka (0.9)\n",
      "boksen -> boken (0.9090909090909091)\n",
      "razmislati -> razmisljati (0.9523809523809523)\n",
      "pomislti -> pomisliti (0.9411764705882353)\n",
      "istrgati -> izstrgati (0.9411764705882353)\n",
      "miski -> omiski (0.9090909090909091)\n",
      "strje -> strtje (0.9090909090909091)\n",
      "vziveti -> vzziveti (0.9333333333333333)\n",
      "frajar -> frajtar (0.9230769230769231)\n",
      "ostal -> postal (0.9090909090909091)\n",
      "ostal -> postal (0.9090909090909091)\n",
      "navelicati -> znavelicati (0.9523809523809523)\n",
      "uprasanje -> prasanje (0.9411764705882353)\n",
      "navciti -> naviti (0.9230769230769231)\n",
      "preoblek -> preobleka (0.9411764705882353)\n",
      "pulcati -> pucati (0.9230769230769231)\n",
      "alojza -> lojza (0.9090909090909091)\n",
      "alojza -> lojza (0.9090909090909091)\n",
      "vziveti -> vzziveti (0.9333333333333333)\n",
      "vzgodba -> zgodba (0.9230769230769231)\n",
      "ostal -> postal (0.9090909090909091)\n",
      "hihih -> hihihi (0.9090909090909091)\n",
      "alojza -> lojza (0.9090909090909091)\n",
      "sprasevati -> prasevati (0.9473684210526315)\n",
      "mogoca -> mogoc (0.9090909090909091)\n",
      "kokosje -> okosje (0.9230769230769231)\n",
      "mogoca -> mogoc (0.9090909090909091)\n",
      "lahek -> plahek (0.9090909090909091)\n",
      "banje -> branje (0.9090909090909091)\n",
      "obcuek -> obcutek (0.9230769230769231)\n",
      "preprican -> prepricevan (0.9)\n",
      "kokoma -> skokoma (0.9230769230769231)\n",
      "kokosje -> okosje (0.9230769230769231)\n",
      "barvaza -> baraza (0.9230769230769231)\n",
      "friend -> frend (0.9090909090909091)\n",
      "tuneti -> tutneti (0.9230769230769231)\n",
      "gospodicen -> gospodicna (0.9)\n",
      "igotov -> gotov (0.9090909090909091)\n",
      "odgoveor -> odgovor (0.9333333333333333)\n",
      "brcen -> ibrcen (0.9090909090909091)\n",
      "obcutka -> obcutkar (0.9333333333333333)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "adeti -> zadeti (0.9090909090909091)\n",
      "pokvarjen -> pokvarjenka (0.9)\n",
      "smucat -> smucati (0.9230769230769231)\n",
      "neomen -> nepomen (0.9230769230769231)\n",
      "izracunati -> izracunjati (0.9523809523809523)\n",
      "preiskusiti -> presusiti (0.9)\n",
      "astronavta -> astronavtka (0.9523809523809523)\n",
      "predcednik -> predsednik (0.9)\n",
      "programera -> programerka (0.9523809523809523)\n",
      "tiina -> tisina (0.9090909090909091)\n",
      "poskusiti -> posusiti (0.9411764705882353)\n",
      "pripet -> priplet (0.9230769230769231)\n",
      "potruditi -> potrditi (0.9411764705882353)\n",
      "ymotiti -> motiti (0.9230769230769231)\n",
      "italio -> italo (0.9090909090909091)\n",
      "priduziti -> pridruziti (0.9473684210526315)\n",
      "preprican -> prepricevan (0.9)\n",
      "roast -> rozast (0.9090909090909091)\n",
      "sprasevati -> prasevati (0.9473684210526315)\n",
      "nuciti -> vnuciti (0.9230769230769231)\n",
      "hihih -> hihihi (0.9090909090909091)\n",
      "bolje -> boljek (0.9090909090909091)\n",
      "menina -> zmenina (0.9230769230769231)\n",
      "skakkati -> skakati (0.9333333333333333)\n",
      "tanek -> tanrek (0.9090909090909091)\n",
      "fusbati -> fusati (0.9230769230769231)\n",
      "zaljubljen -> zaljubljenka (0.9090909090909091)\n",
      "zogati -> zorgati (0.9230769230769231)\n",
      "gdati -> gadati (0.9090909090909091)\n",
      "prebra-ti -> prebrati (0.9411764705882353)\n",
      "preprican -> prepricevan (0.9)\n",
      "brina -> brzina (0.9090909090909091)\n",
      "brina -> brzina (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "jsaditi -> saditi (0.9230769230769231)\n",
      "srcka -> srecka (0.9090909090909091)\n",
      "kristijan -> kristjan (0.9411764705882353)\n",
      "kristijan -> kristjan (0.9411764705882353)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "banan -> banjan (0.9090909090909091)\n",
      "banan -> banjan (0.9090909090909091)\n",
      "brina -> brzina (0.9090909090909091)\n",
      "banati -> branati (0.9230769230769231)\n",
      "unati -> uznati (0.9090909090909091)\n",
      "pojda -> pojeda (0.9090909090909091)\n",
      "namenjen -> namrenjen (0.9411764705882353)\n",
      "pisti -> pistil (0.9090909090909091)\n",
      "govort -> govor (0.9090909090909091)\n",
      "salati -> spalati (0.9230769230769231)\n",
      "lazanje -> blazanje (0.9333333333333333)\n",
      "brawti -> brati (0.9090909090909091)\n",
      "zacen -> zracen (0.9090909090909091)\n",
      "milant -> milan (0.9090909090909091)\n",
      "patrik -> paterik (0.9230769230769231)\n",
      "letec -> pletec (0.9090909090909091)\n",
      "letec -> pletec (0.9090909090909091)\n",
      "letec -> pletec (0.9090909090909091)\n",
      "letec -> pletec (0.9090909090909091)\n",
      "nuciti -> vnuciti (0.9230769230769231)\n",
      "vzvati -> vzvrati (0.9230769230769231)\n",
      "vzival -> zival (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "aditi -> zaditi (0.9090909090909091)\n",
      "vavdnik -> vadnik (0.9230769230769231)\n",
      "napiss -> napis (0.9090909090909091)\n",
      "mogoca -> mogoc (0.9090909090909091)\n",
      "hlbec -> hlebec (0.9090909090909091)\n",
      "upisati -> pisati (0.9230769230769231)\n",
      "domislija -> domisljija (0.9473684210526315)\n",
      "ljuda -> sljuda (0.9090909090909091)\n",
      "spremeniv -> spremenljiv (0.9)\n",
      "resnoo -> resnook (0.9230769230769231)\n",
      "vodnika -> vodenika (0.9333333333333333)\n",
      "unicen -> udnicen (0.9230769230769231)\n",
      "zgdba -> zgodba (0.9090909090909091)\n",
      "unicen -> udnicen (0.9230769230769231)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "spodbujati -> podbujati (0.9473684210526315)\n",
      "lovlenje -> lovljenje (0.9411764705882353)\n",
      "predstavljen -> predstavljenje (0.9230769230769231)\n",
      "fantazisk -> fantazijski (0.9)\n",
      "haloo -> halooo (0.9090909090909091)\n",
      "skaca -> skalca (0.9090909090909091)\n",
      "hahah -> hahaha (0.9090909090909091)\n",
      "hahah -> hahaha (0.9090909090909091)\n",
      "nauca -> naucba (0.9090909090909091)\n",
      "apaka -> napaka (0.9090909090909091)\n",
      "nauca -> naucba (0.9090909090909091)\n",
      "albina -> albinka (0.9230769230769231)\n",
      "naucit -> nauciti (0.9230769230769231)\n",
      "zivati -> zrivati (0.9230769230769231)\n",
      "clovekk -> clovek (0.9230769230769231)\n",
      "albina -> albinka (0.9230769230769231)\n",
      "albina -> albinka (0.9230769230769231)\n",
      "vvodnik -> vzvodnik (0.9333333333333333)\n",
      "albina -> albinka (0.9230769230769231)\n",
      "albina -> albinka (0.9230769230769231)\n",
      "albina -> albinka (0.9230769230769231)\n",
      "rovsnik -> rovnik (0.9230769230769231)\n",
      "hihih -> hihihi (0.9090909090909091)\n",
      "rovsnik -> rovnik (0.9230769230769231)\n",
      "hihihihih -> hihihihi (0.9411764705882353)\n",
      "teljica -> steljica (0.9333333333333333)\n",
      "vpraviciti -> praviciti (0.9473684210526315)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "navezan -> naveza (0.9230769230769231)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "zalostn -> zalosten (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "navezan -> naveza (0.9230769230769231)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "spremeniti -> premeniti (0.9473684210526315)\n",
      "posti -> sposti (0.9090909090909091)\n",
      "elina -> zelina (0.9090909090909091)\n",
      "bedenje -> bredenje (0.9333333333333333)\n",
      "brina -> brzina (0.9090909090909091)\n",
      "adijos -> adijo (0.9090909090909091)\n",
      "potrebuje -> potrebusje (0.9473684210526315)\n",
      "konca -> skonca (0.9090909090909091)\n",
      "hojje -> hojdje (0.9090909090909091)\n",
      "komor -> komora (0.9090909090909091)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "zalos -> zalost (0.9090909090909091)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "haloo -> halooo (0.9090909090909091)\n",
      "konca -> skonca (0.9090909090909091)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "ostal -> postal (0.9090909090909091)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "zelisa -> elisa (0.9090909090909091)\n",
      "karina -> okarina (0.9230769230769231)\n",
      "musti -> musiti (0.9090909090909091)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "posloviti -> poosloviti (0.9473684210526315)\n",
      "alelelujati -> alelujati (0.9)\n",
      "nardeiti -> nardeciti (0.9411764705882353)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "brava -> brlava (0.9090909090909091)\n",
      "izpustiti -> izustiti (0.9411764705882353)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "drugaca -> druzgaca (0.9333333333333333)\n",
      "odsti -> odsuti (0.9090909090909091)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "preprican -> prepricevan (0.9)\n",
      "varjeten -> vajeten (0.9333333333333333)\n",
      "pozdravta -> pozdrava (0.9411764705882353)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "odsti -> odsuti (0.9090909090909091)\n",
      "izgubljen -> izguljen (0.9411764705882353)\n",
      "rozti -> roziti (0.9090909090909091)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "pocutiti -> ocutiti (0.9333333333333333)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "conversations_corr = [[spelling_correction.replace_if_close(token) for token in tokens] for tokens in tqdm(conversations)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Token grouping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T14:25:07.261020Z",
     "start_time": "2020-04-25T14:25:07.250090Z"
    }
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:44:14.579590Z",
     "start_time": "2020-04-25T12:44:14.552614Z"
    }
   },
   "outputs": [],
   "source": [
    "import string\n",
    "\n",
    "class GibberishDetector():\n",
    "    def __init__(self, roof_removal):\n",
    "        \"\"\"Gibberish detector trained on a dictionary of real words (using Markov chains).\"\"\"\n",
    "        \n",
    "        self.invalid_regex = re.compile('[^a-z]')\n",
    "        \n",
    "        self.states = string.ascii_lowercase + '*^$'\n",
    "        self.state_index = {char: i for i, char in enumerate(self.states)}\n",
    "        \n",
    "        self.probs = np.zeros((len(self.states), len(self.states)))\n",
    "        self.threshold = 0.0\n",
    "        \n",
    "        self.roof_removal = roof_removal\n",
    "        \n",
    "        \n",
    "    def normalize(self, string):\n",
    "        \"\"\"Replace invalid characters (non-alphabet) with *\"\"\"\n",
    "        \n",
    "        # Remove roofs\n",
    "        string = self.roof_removal.remove(string)\n",
    "        \n",
    "        # Replace out-of-dict chars\n",
    "        string = self.invalid_regex.sub('*', string)\n",
    "        \n",
    "        # Surround with ^ and $\n",
    "        string = '^' + string + '$'\n",
    "        \n",
    "        return string\n",
    "        \n",
    "    def ngram(self, string, n):\n",
    "        \"\"\" Return all n grams from string\"\"\"\n",
    "        \n",
    "        filtered = self.normalize(string)\n",
    "        for start in range(0, len(filtered) - n + 1):\n",
    "            yield ''.join(filtered[start:start + n])\n",
    "            \n",
    "    def train(self, dictionary_txt, good_txt, bad_txt):\n",
    "        counts = np.zeros((len(self.states), len(self.states)))\n",
    "        \n",
    "        with open(dictionary_txt, 'r') as file:\n",
    "            word_iter = (self.normalize(line.strip()) for line in file)\n",
    "        \n",
    "            for word in word_iter:\n",
    "                for c1, c2 in self.ngram(word, 2):\n",
    "                    c1i = self.state_index[c1]\n",
    "                    c2i = self.state_index[c2]\n",
    "\n",
    "                    counts[c1i, c2i] += 1    \n",
    "                    \n",
    "        \n",
    "        # Add small probability even to missing transitions\n",
    "        laplace_vector = np.maximum((counts.sum(axis=1)*0.01/len(counts)), 1)[:, np.newaxis]\n",
    "        counts = np.maximum(counts, laplace_vector)\n",
    "        \n",
    "        # Compute log probabilities\n",
    "        sums = counts.sum(axis=1)[:, np.newaxis]\n",
    "        self.probs = np.log(counts / sums)\n",
    "        \n",
    "        # Compute best threshold\n",
    "        with open(good_txt, 'r') as file:\n",
    "            word_iter = (line.strip() for line in file)\n",
    "            good_probs = np.array([self.word_probability(word) for word in word_iter])\n",
    "        \n",
    "        with open(bad_txt, 'r') as file:\n",
    "            word_iter = (line.strip() for line in file)\n",
    "            bad_probs = np.array([self.word_probability(word) for word in word_iter])\n",
    "        \n",
    "        min_g = np.min(good_probs)\n",
    "        max_b = np.max(bad_probs)\n",
    "        \n",
    "        self.threshold = (min_g + max_b) * 0.5\n",
    "        \n",
    "        # Test threshold\n",
    "        print(f'Correct good: {np.mean(good_probs > self.threshold)}')\n",
    "        print(f'Correct bad: {np.mean(bad_probs <= self.threshold)}')\n",
    "            \n",
    "    def word_probability(self, word):\n",
    "        word = self.normalize(word)\n",
    "        \n",
    "        log_prob = 0.0\n",
    "        count = 0\n",
    "        for c1, c2 in self.ngram(word, 2):\n",
    "            c1i = self.state_index[c1]\n",
    "            c2i = self.state_index[c2]\n",
    "\n",
    "            log_prob += self.probs[c1i, c2i]\n",
    "            count += 1\n",
    "            \n",
    "        return np.exp(log_prob / count)\n",
    "        \n",
    "    def is_gibberish(self, word):\n",
    "        return self.word_probability(word) <= self.threshold\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:44:23.914257Z",
     "start_time": "2020-04-25T12:44:14.967679Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct good: 1.0\n",
      "Correct bad: 1.0\n"
     ]
    }
   ],
   "source": [
    "gibberish_detector = GibberishDetector(roof_removal)\n",
    "gibberish_detector.train('data/dict-sl.txt', 'data/gibberish_good.txt', 'data/gibberish_bad.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T12:44:26.457431Z",
     "start_time": "2020-04-25T12:44:26.448845Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, False)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gibberish_detector.is_gibberish('kagfjhnjzguyd'), gibberish_detector.is_gibberish('otorinolaringolog')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T14:25:10.182064Z",
     "start_time": "2020-04-25T14:25:10.163465Z"
    }
   },
   "outputs": [],
   "source": [
    "# Regex for symbols etc.\n",
    "other_regex = re.compile(r'[^\\w?!.,-]')\n",
    "\n",
    "def group_tokens(token):\n",
    "    \n",
    "    if token.isdigit():\n",
    "        print(f'<number> <- {token}')\n",
    "        token = '<number>'\n",
    "    elif len(token) > 4 and gibberish_detector.is_gibberish(token):\n",
    "        print(f'<gibberish> <- {token}')\n",
    "        token = '<gibberish>'\n",
    "    elif other_regex.search(token) is not None:\n",
    "        print(f'<other> <- {token}')\n",
    "        token = '<other>'\n",
    "        \n",
    "    return token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T14:25:10.749650Z",
     "start_time": "2020-04-25T14:25:10.457040Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<gibberish> <- ooonnnee\n",
      "<gibberish> <- zzzaaakkkaaajjje\n",
      "<gibberish> <- vvvtttfff\n",
      "<number> <- 24\n",
      "<gibberish> <- uijkh\n",
      "<number> <- 3\n",
      "<gibberish> <- wehrw\n",
      "<gibberish> <- ebzztdftzdztdtziztdti\n",
      "<number> <- 45\n",
      "<gibberish> <- wweddfdfffdfbfbdrez\n",
      "<number> <- 5\n",
      "<gibberish> <- rtrthfhgf\n",
      "<gibberish> <- fthfhfthfthfthfthfthfhfghfthzfrtje\n",
      "<gibberish> <- gz---aw\n",
      "<gibberish> <- wawdwawsdawd\n",
      "<gibberish> <- trrreo\n",
      "<gibberish> <- wdawsdasdssawww\n",
      "<gibberish> <- awdadawdwdawdawdadjadvjawgdagdhasxbasjkhdkawbkjasgdawghdkaghdkajkgawkgd\n",
      "<gibberish> <- kagfjhgsfvawgdadjzgfinjzjgujlizuukgguyd\n",
      "<number> <- 76\n",
      "<gibberish> <- uzfrhtthdhjdftdtttzezgzuutttzzzrdtdzka\n",
      "<gibberish> <- ktfgca\n",
      "<gibberish> <- iluehz\n",
      "<gibberish> <- hsehfiwhf\n",
      "<gibberish> <- usrhfsiufhweufhwekhsehfkshuefksuf\n",
      "<gibberish> <- kufhskuhzezhshkeufszehwehzefhs\n",
      "<gibberish> <- fhwefefsefsefsfesfeww\n",
      "<gibberish> <- sssohgkgghuocg\n",
      "<gibberish> <- sekhsoiefheiohfioehfehfrehkflwefhlief\n",
      "<gibberish> <- erilceilzwhreibaeizcsfhsezfzscjsgweuwtrewfasegwegerrrsjhfesgvfkshefjiseifhsuieffnciejkfsaocejkfw-ckfwefjoisejliw\n",
      "<gibberish> <- dbfsfuoisurg\n",
      "<number> <- 4\n",
      "<number> <- 0\n",
      "<gibberish> <- faehfgydlfhkskdfjskjgnsdvfnysjvfnsdka\n",
      "<gibberish> <- eglsirhulghsbfjsdr\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<gibberish> <- lllooojjje\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<number> <- 24\n",
      "<number> <- 4\n",
      "<gibberish> <- dgfvhvdccgflyegyvuvba\n",
      "<number> <- 6\n",
      "<number> <- 16\n",
      "<number> <- 6\n",
      "<number> <- 3\n",
      "<gibberish> <- jhrpig\n",
      "<other> <- [\n",
      "<gibberish> <- ovnjgyuag\n",
      "<number> <- 8467\n",
      "<number> <- 78\n",
      "<gibberish> <- tnhdfrbn\n",
      "<number> <- 04\n",
      "<number> <- 9\n",
      "<number> <- 4\n",
      "<number> <- 9\n",
      "<other> <- [\n",
      "<number> <- 34\n",
      "<number> <- 4\n",
      "<gibberish> <- uwtoaba\n",
      "<number> <- 9057\n",
      "<number> <- 0\n",
      "<gibberish> <- qwertzuiopsdzzcclkjhgfdsayxcvbnm\n",
      "<gibberish> <- odkkwejwjksjeziowzoehwejwweojqojrioj\n",
      "<gibberish> <- jdfguh\n",
      "<number> <- 7\n",
      "<gibberish> <- trhgwugf\n",
      "<number> <- 78034\n",
      "<number> <- 4\n",
      "<gibberish> <- fkcvnwvzx\n",
      "<number> <- 4\n",
      "<number> <- 90\n",
      "<number> <- 95\n",
      "<number> <- 5\n",
      "<number> <- 0\n",
      "<number> <- 49377\n",
      "<gibberish> <- xcklbti\n",
      "<number> <- 04\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<other> <- <\n",
      "<gibberish> <- ablllbbbm\n",
      "<gibberish> <- pppiio\n",
      "<gibberish> <- njdnkfbwdhibfusddfhbwkefqwuehuitejqwgfeocruetuearzhopizqhaetorjqwrhiotjhp\n",
      "<other> <- <\n",
      "<gibberish> <- erawafohsdoidjopwaeodf\n",
      "<gibberish> <- sdkkdasjkldhawdhjixwdhdedebhdebklcjzulpjicdsdhuwejfrtrjigtrjigtjigtjgtjigtrjugrjdjfu\n",
      "<number> <- 9\n",
      "<number> <- 8\n",
      "<number> <- 4\n",
      "<gibberish> <- idfpvefh\n",
      "<gibberish> <- whipfeipfijpnwrjiopwrcfkopvfrrkomjovfkvgftvkovgftrjioctttjujioerjioeijorejioejioermrorrjiogtrfgftrjigftrrrgtr\n",
      "<gibberish> <- nklvf\n",
      "<gibberish> <- nlmklmmkvfbtrnmobtrniopbtrgtrbgtrbvgtbvgrjiovgrjiovgtjivgtrvggjipvgjievgfjievgjilvfbhjfhvgfca\n",
      "<gibberish> <- jfjdjakhdoasjfkasjdfdcjhcprigweio\n",
      "<number> <- 6\n",
      "<number> <- 8\n",
      "<gibberish> <- rijrfeoiruowdjakskgjsldaksckalsjaflcd\n",
      "<gibberish> <- dskfmkdhsjxehliptosdauis\n",
      "<number> <- 7\n",
      "<number> <- 4\n",
      "<gibberish> <- kkgcdf\n",
      "<gibberish> <- cycdlpq\n",
      "<number> <- 2\n",
      "<number> <- 24\n",
      "<gibberish> <- ghzdgtserhdztti\n",
      "<gibberish> <- fvjxsfjkasodkpsfca\n",
      "<number> <- 6\n",
      "<number> <- 6\n",
      "<number> <- 22\n",
      "<number> <- 25\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<other> <- ¤\n",
      "<number> <- 7\n",
      "<number> <- 26\n",
      "<number> <- 19\n",
      "<gibberish> <- najtrdnejĺˇ\n",
      "<number> <- 7\n",
      "<gibberish> <- hujĺˇ\n",
      "<number> <- 26\n",
      "<other> <- ’\n",
      "<other> <- ©\n",
      "<number> <- 5\n",
      "<number> <- 15\n",
      "<other> <- \"\n",
      "<other> <- \"\n",
      "<number> <- 6\n",
      "<number> <- 11122333444\n",
      "<gibberish> <- napisssiio\n",
      "<number> <- 18\n",
      "<number> <- 16\n",
      "<gibberish> <- gshnslkjda\n",
      "<gibberish> <- oksferwkglfklgfmlfti\n",
      "<gibberish> <- nsdzdzcscc\n",
      "<gibberish> <- zfhkuwujlzierouoppszrzzzgozg\n",
      "<number> <- 9\n",
      "<gibberish> <- oqwksoksoq\n",
      "<number> <- 2\n",
      "<gibberish> <- mkwdfn\n",
      "<number> <- 54\n",
      "<number> <- 67\n",
      "<gibberish> <- uoewtzeoq\n",
      "<number> <- 8\n",
      "<gibberish> <- hjeujs\n",
      "<gibberish> <- eofue\n",
      "<number> <- 9\n",
      "<number> <- 7904\n",
      "<gibberish> <- chglwfklcti\n",
      "<number> <- 12345676890\n",
      "<other> <- '\n",
      "<number> <- 22\n",
      "<gibberish> <- tredgrflhxnfgifn\n",
      "<number> <- 3\n",
      "<number> <- 5\n",
      "<number> <- 10\n",
      "<number> <- 11\n",
      "<gibberish> <- hlgwhwtjka\n",
      "<gibberish> <- tztzw\n",
      "<gibberish> <- gffgr\n",
      "<gibberish> <- rubgjsbrsdgjtidflurhjfir\n",
      "<gibberish> <- gqerza\n",
      "<number> <- 25\n",
      "<number> <- 8\n",
      "<number> <- 8\n",
      "<number> <- 25\n",
      "<number> <- 25\n",
      "<number> <- 8\n",
      "<number> <- 17\n",
      "<number> <- 17\n",
      "<number> <- 25\n",
      "<number> <- 21\n",
      "<gibberish> <- zgsxeghxba\n",
      "<gibberish> <- hbdfbjvhngjbhng\n",
      "<number> <- 21\n",
      "<number> <- 21\n",
      "<gibberish> <- cacacacaccghfvuti\n",
      "<number> <- 4\n",
      "<gibberish> <- hgjma\n",
      "<number> <- 45\n",
      "<gibberish> <- oithje\n",
      "<number> <- 76\n",
      "<gibberish> <- oihjoitjgotkjgotzkhclokupjok\n",
      "<number> <- 7\n",
      "<gibberish> <- opjhzkzohkzohkopzkhzkhopkhozkhopzkhozokhozkhokzohkzopkmxdca\n",
      "<gibberish> <- elotgtg\n",
      "<gibberish> <- fogtkigoztkightgzoktigkfgjhou\n",
      "<number> <- 9\n",
      "<gibberish> <- poktgtpohpzpsupzpha\n",
      "<number> <- 0\n",
      "<number> <- 0\n",
      "<number> <- 666\n",
      "<number> <- 21\n",
      "<number> <- 25\n",
      "<gibberish> <- tzp'cujg\n",
      "<number> <- 4\n",
      "<number> <- 5\n",
      "<number> <- 5\n",
      "<number> <- 9\n",
      "<number> <- 54\n",
      "<number> <- 9\n",
      "<number> <- 5\n",
      "<number> <- 89\n",
      "<number> <- 9000\n",
      "<gibberish> <- dhjhakhh\n",
      "<gibberish> <- hihihihihihihihihihihihihihihihihihiihhihiihihhihjiihihhiihhiihhihiihihihihih\n",
      "<gibberish> <- wwwqweqwertz\n",
      "<number> <- 4\n",
      "<number> <- 19\n",
      "<number> <- 20\n",
      "<number> <- 4\n",
      "<number> <- 1\n",
      "<gibberish> <- kakkakakaq\n",
      "<number> <- 5\n",
      "<number> <- 4\n",
      "<number> <- 3\n",
      "<number> <- 7\n",
      "<gibberish> <- lahkp\n",
      "<gibberish> <- copkp\n",
      "<other> <- ¸\n",
      "<gibberish> <- dddooobbbeeerrr\n",
      "<other> <- '\n",
      "<other> <- '\n",
      "<other> <- '\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<gibberish> <- hewca\n",
      "<other> <- *\n",
      "<gibberish> <- heeec\n",
      "<gibberish> <- heeec\n",
      "<gibberish> <- heeec\n",
      "<gibberish> <- heeec\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<number> <- 20\n",
      "<number> <- 5\n",
      "<gibberish> <- heeec\n",
      "<gibberish> <- heeec\n",
      "<gibberish> <- heeec\n",
      "<number> <- 15\n",
      "<gibberish> <- jasss\n",
      "<other> <- /\n",
      "<other> <- :\n",
      "<number> <- 11\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- :\n",
      "<other> <- :D\n",
      "<gibberish> <- sorry\n",
      "<other> <- *\n",
      "<other> <- /\n",
      "<number> <- 26\n",
      "<number> <- 1\n",
      "<number> <- 10\n",
      "<number> <- 1\n",
      "<other> <- *\n",
      "<number> <- 15\n",
      "<number> <- 4\n",
      "<number> <- 9\n",
      "<other> <- *\n",
      "<other> <- :)\n",
      "<other> <- )\n",
      "<other> <- )\n",
      "<other> <- (:\n",
      "<other> <- :\n",
      "<number> <- 3\n",
      "<other> <- #\n",
      "<other> <- #\n",
      "<other> <- #tihobodti\n",
      "<gibberish> <- #cudnfejs\n",
      "<gibberish> <- uuutttiiihhhnnniio\n",
      "<number> <- 11\n",
      "<other> <- :\n",
      "<number> <- 3\n",
      "<number> <- 23\n",
      "<number> <- 20\n",
      "<other> <- \"\n",
      "<number> <- 23\n",
      "<other> <- \"\n",
      "<number> <- 20\n",
      "<other> <- \"\n",
      "<other> <- \"\n",
      "<other> <- \"\n",
      "<number> <- 25\n",
      "<other> <- \"\n",
      "<number> <- 11\n",
      "<other> <- :)\n",
      "<gibberish> <- kkkdddoo\n",
      "<gibberish> <- jjjee\n",
      "<gibberish> <- fffrrraaannncccaaaggg\n",
      "<number> <- 222555\n",
      "<number> <- 14\n",
      "<other> <- :)\n",
      "<gibberish> <- nherfnhrgevhnbzrfevnhrefvnhbzuefrvbzuhnevfbuevbuvebnzhnbvenhvenouxdmjkzgaf\n",
      "<number> <- 4\n",
      "<gibberish> <- REMIXFR\n",
      "<number> <- 4\n",
      "<gibberish> <- ATEDS\n",
      "<number> <- 3\n",
      "<gibberish> <- UHDYSOG\n",
      "<number> <- 8634\n",
      "<gibberish> <- ARCXHMOU\n",
      "<number> <- 8\n",
      "<gibberish> <- DFCXOSA\n",
      "<number> <- 863\n",
      "<gibberish> <- CTJRU\n",
      "<gibberish> <- dfoxg\n",
      "<number> <- 86\n",
      "<number> <- 4\n",
      "<number> <- 78\n",
      "<gibberish> <- oookasyrs\n",
      "<number> <- 3\n",
      "<number> <- 4\n",
      "<number> <- 38\n",
      "<gibberish> <- fxszti\n",
      "<number> <- 87\n",
      "<number> <- 4\n",
      "<gibberish> <- xhgtzdresf\n",
      "<number> <- 4\n",
      "<gibberish> <- utferdwsg\n",
      "<gibberish> <- htzmuefrdwgutzmefrdwturfwdtgzferdutgzfredsgzfdser\n",
      "<other> <- >\n",
      "<other> <- <]<>\n",
      "<other> <- §\n",
      "<other> <- ]\n",
      "<gibberish> <- gbtfrce\n",
      "<number> <- 4\n",
      "<gibberish> <- hbfrdewasnhbzufaxdca\n",
      "<gibberish> <- xdcsen\n",
      "<gibberish> <- hbedfrcvn\n",
      "<gibberish> <- gedrcf\n",
      "<gibberish> <- nedcrsxnjmedrfcccsss\n",
      "<gibberish> <- pppiiisssmmmoo\n",
      "<gibberish> <- rrrooosssnnnoo\n",
      "<gibberish> <- aaadddiiijjjoo\n",
      "<number> <- 4\n",
      "<number> <- 18\n",
      "<number> <- 11\n",
      "<number> <- 15\n",
      "<other> <- :)\n",
      "<number> <- 15\n",
      "<number> <- 15\n",
      "<number> <- 15\n",
      "<number> <- 2\n",
      "<number> <- 15\n",
      "<other> <- :D\n",
      "<number> <- 8\n",
      "<number> <- 15\n",
      "<other> <- :)\n",
      "<other> <- :D\n",
      "<number> <- 4\n",
      "<number> <- 4\n",
      "<number> <- 7\n",
      "<number> <- 4\n",
      "<number> <- 7\n",
      "<number> <- 7\n",
      "<number> <- 14\n",
      "<other> <- :)\n",
      "<number> <- 15\n",
      "<other> <- :D\n",
      "<number> <- 12\n",
      "<number> <- 12\n",
      "<number> <- 12\n",
      "<number> <- 1\n",
      "<number> <- 2\n",
      "<number> <- 6\n",
      "<other> <- \"\n",
      "<other> <- \"\n",
      "<gibberish> <- cvfnvnvvcvvnfv\n",
      "<gibberish> <- cmdjfdanf\n",
      "<gibberish> <- vynyf\n",
      "<gibberish> <- jvalkvbkjy\n",
      "<gibberish> <- bgchjhfibfhfbvgg\n",
      "<gibberish> <- chbnbgkbhg\n",
      "<gibberish> <- jnzcjbgnvuvnje\n",
      "<gibberish> <- djhsbf\n",
      "<gibberish> <- cjhbojfhefhoejicdwhkldnciwccmdvbdnbfvccsbdnjscdbdjhctlvgzjje\n",
      "<gibberish> <- jjjhjjjksnkhlsfdaojdzgkesrvnfljcwrojgnbgoernbgjfnsf\n",
      "<gibberish> <- bmegvnf\n",
      "<gibberish> <- jglgrnbcs\n",
      "<gibberish> <- jdxbng\n",
      "<gibberish> <- mydvngscontjeahv\n",
      "<gibberish> <- nesfvklnmkrfd\n",
      "<gibberish> <- abkcsgvnvvpsnegf\n",
      "<gibberish> <- afmcalnkfveacfwacnljbkcfjvsavvfavfdavasfjnlcdvfacjlba\n",
      "<gibberish> <- vdwajlcnacbiochwdcaio\n",
      "<gibberish> <- cevbrljquecrgnciaekzsmczwrlm\n",
      "<gibberish> <- letgwetgwgrewnccen\n",
      "<gibberish> <- ivcnhfkbpnnkgjfd\n",
      "<gibberish> <- gcmzgxngfxfhgxn\n",
      "<gibberish> <- heeej\n",
      "<number> <- 1\n",
      "<gibberish> <- gufghhdhw\n",
      "<number> <- 5\n",
      "<number> <- 4\n",
      "<number> <- 8\n",
      "<gibberish> <- hahhahahahahahahahahah\n",
      "<gibberish> <- gsgjfgesrf\n",
      "<number> <- 46\n",
      "<number> <- 384\n",
      "<number> <- 3\n",
      "<number> <- 3\n",
      "<gibberish> <- ruifzzerhtrieruegruzzzrrruuurgffgsuzf\n",
      "<gibberish> <- efgefgzfsgjl\n",
      "<number> <- 47\n",
      "<gibberish> <- gzjfgeejsfgvjegfjgfgfrzfrkfkffzsfszfsjfhsfsfjfgsf\n",
      "<gibberish> <- vjguz\n",
      "<number> <- 4\n",
      "<gibberish> <- zrgdetrtrugfgrkhkfhdhkfhhtkrkhrhkfzfhskghkggljgorrt\n",
      "<number> <- 8\n",
      "<number> <- 678777\n",
      "<gibberish> <- hkhgkhgkdjg\n",
      "<gibberish> <- hfgetrdf\n",
      "<number> <- 4646575756\n",
      "<number> <- 5\n",
      "<gibberish> <- hvbvmfdudhvsfhffbsslhsghkgsfsehdfnsfsfjlkakgkkfgkfsfsfjsffkggjshjhjsfsgffgf\n",
      "<gibberish> <- gugerwgt\n",
      "<gibberish> <- gryjfgkffgdkduyibthdgchxixx\n",
      "<number> <- 13\n",
      "<number> <- 24\n",
      "<other> <- $\n",
      "<other> <- $\n",
      "<other> <- #\n",
      "<gibberish> <- jfnhvjkbnje\n",
      "<gibberish> <- gnjjhnbka\n",
      "<gibberish> <- bnkfjjkgmfkjgjfhje\n",
      "<number> <- 8576587478\n",
      "<number> <- 4775\n",
      "<gibberish> <- ndbhje\n",
      "<gibberish> <- cfbjfr\n",
      "<number> <- 8\n",
      "<number> <- 587\n",
      "<number> <- 12\n",
      "<gibberish> <- napisalrdzrdthzfkghfgztrdtzserwsztjzftvuthdftzkf\n",
      "<number> <- 12341243565\n",
      "<number> <- 7\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- #\n",
      "<gibberish> <- idddej\n",
      "<number> <- 19\n",
      "<gibberish> <- wiiiwwwiio\n",
      "<other> <- \"\n",
      "<other> <- \"\n",
      "<gibberish> <- hiihiihihihihihihahahahahahahuhuhuhuhumumumum\n",
      "<number> <- 12345678\n",
      "<number> <- 90\n",
      "<gibberish> <- dzspoiuztrewqasdfghjklccz\n",
      "<other> <- <\n",
      "<gibberish> <- yxcvbn\n",
      "<gibberish> <- hgtrfhfrrirguhvdjgwiti\n",
      "<gibberish> <- hgjfhfgsdvghjda\n",
      "<number> <- 8\n",
      "<gibberish> <- sghfuhgsfhgkljsfhgkljshfkjgizgwerj\n",
      "<number> <- 9\n",
      "<gibberish> <- fiweeruh\n",
      "<number> <- 8\n",
      "<number> <- 8\n",
      "<gibberish> <- kaaajjje\n",
      "<number> <- 8\n",
      "<number> <- 23\n",
      "<number> <- 17\n",
      "<number> <- 17\n",
      "<gibberish> <- sxsdfdhtrgttdtrdgfffcsrsgdrscg\n",
      "<number> <- 54\n",
      "<gibberish> <- rgvddfhw\n",
      "<number> <- 3\n",
      "<number> <- 34\n",
      "<number> <- 78\n",
      "<gibberish> <- lahczdz\n",
      "<number> <- 9\n",
      "<other> <- :)\n",
      "<gibberish> <- ffgfshdhgdftgiuzg\n",
      "<gibberish> <- jtiulttuiltutouepfjthgzti\n",
      "<number> <- 500\n",
      "<number> <- 10\n",
      "<number> <- 10\n",
      "<number> <- 13\n",
      "<gibberish> <- vbmvtgtedzkdz\n",
      "<number> <- 5\n",
      "<number> <- 5\n",
      "<gibberish> <- mumumumumumumumumumummumumummumumumumummumumumumummumumumumummuummummumumummumummumumumum\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<gibberish> <- howdy\n",
      "<gibberish> <- imapbooks\n",
      "<number> <- 9\n",
      "<number> <- 8\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<number> <- 9\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- '\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<number> <- 00\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- ~\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- °\n",
      "<other> <- °\n",
      "<other> <- )\n",
      "<gibberish> <- hahahahhahahahahahhahahahhahahahahhahahahahahahahahahahahhahahahahahahahahahahahahahahhaahahahahhahahahahahahahahjahajajahahahahahahahahhahahahahahahahahahahhahaahhahahahahaahhahahahahahahahahahahahahhahah\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<gibberish> <- khukuguigujliuh\n",
      "<number> <- 8\n",
      "<gibberish> <- ighocpjio\n",
      "<number> <- 8\n",
      "<gibberish> <- itgpa\n",
      "<number> <- 8\n",
      "<number> <- 8\n",
      "<number> <- 87674\n",
      "<number> <- 6\n",
      "<gibberish> <- lovcfkghptfhkptlhpctfkhcptkhcsdrflpkjcpdtlg\n",
      "<gibberish> <- peswnjjbggclmcvcti\n",
      "<gibberish> <- cnjtg\n",
      "<gibberish> <- kcgti\n",
      "<gibberish> <- lcfkgfr\n",
      "<gibberish> <- kgfkg\n",
      "<other> <- :)\n",
      "<gibberish> <- cogpegpokcllegplegcrspogrega\n",
      "<gibberish> <- vcdgfkje\n",
      "<gibberish> <- fdcpogr\n",
      "<number> <- 4\n",
      "<number> <- 5\n",
      "<gibberish> <- gcprtz\n",
      "<gibberish> <- urzcnqp\n",
      "<number> <- 9\n",
      "<gibberish> <- tmhpv\n",
      "<other> <- :)\n",
      "<other> <- :)\n",
      "<other> <- :)\n",
      "<other> <- :)\n",
      "<number> <- 5\n",
      "<gibberish> <- grhfbdgd\n",
      "<other> <- :)\n",
      "<gibberish> <- gkvfffchcghh\n",
      "<other> <- :\n",
      "<gibberish> <- sedbfzub\n",
      "<gibberish> <- hkljg\n",
      "<gibberish> <- gglglbghkkfkfkgjkgggkkvmjgggngmgggngjgjggkgggcglglg\n",
      "<gibberish> <- gbghggchkhjg\n",
      "<gibberish> <- gggkjkjik\n",
      "<other> <- :)\n",
      "<other> <- p'\n",
      "<other> <- >\n",
      "<other> <- :)\n",
      "<other> <- :)\n",
      "<gibberish> <- jgnggfjnblfvfnhkljkbdoskhmkvrgjoewrtihhr\n",
      "<gibberish> <- hjcgzuevewugfrtffr\n",
      "<number> <- 6\n",
      "<number> <- 4\n",
      "<number> <- 56\n",
      "<gibberish> <- zrerueiwztf\n",
      "<number> <- 2\n",
      "<number> <- 9\n",
      "<number> <- 22\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<gibberish> <- nfjhvaeuirhoieiura\n",
      "<number> <- 23\n",
      "<other> <- /\n",
      "<number> <- 13\n",
      "<other> <- /\n",
      "<other> <- '\n",
      "<other> <- '\n",
      "<gibberish> <- hhaahhhaahaha\n",
      "<other> <- :p\n",
      "<number> <- 10\n",
      "<other> <- ;\n",
      "<other> <- ;\n",
      "<other> <- :\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<other> <- ;)\n",
      "<number> <- 0\n",
      "<number> <- 0\n",
      "<other> <- '\n",
      "<other> <- '\n",
      "<other> <- '\n",
      "<other> <- '\n",
      "<gibberish> <- ahahahahahahahahahahahahahahaahhahahahahaaha\n",
      "<gibberish> <- uvbyjvbiufoefnkjnaefofgaeoehogeaoddgfioshsrpa\n",
      "<other> <- :)\n",
      "<gibberish> <- kajjje\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<gibberish> <- mumhhaa\n",
      "<gibberish> <- kajjje\n",
      "<gibberish> <- qfcukqehg\n",
      "<gibberish> <- malikkk\n",
      "<gibberish> <- ahddd\n",
      "<gibberish> <- ofeuwfbcwenfjkdnwemjcfn\n",
      "<gibberish> <- askmxeopmdwnkcfckwqrwd\n",
      "<gibberish> <- nkdsnkfcnweklcvfnlwcsca\n",
      "<gibberish> <- nejjjccca\n",
      "<gibberish> <- jdeel\n",
      "<number> <- 18\n",
      "<gibberish> <- lamamammakyjyy\n",
      "<number> <- 5\n",
      "<number> <- 0\n",
      "<gibberish> <- edhenhcdwnajcfjbswdbebwfjbcfknekcksdbacfjlbdejcmdnfhdsce\n",
      "<number> <- 6\n",
      "<gibberish> <- qkhwdckhbqercjlvqejd\n",
      "<gibberish> <- vqhsljda\n",
      "<gibberish> <- ghjvewcwgejlf\n",
      "<number> <- 6\n",
      "<gibberish> <- fxmqsxcamjscf\n",
      "<gibberish> <- qkjwfd\n",
      "<gibberish> <- kuqfwkga\n",
      "<other> <- &\n",
      "<other> <- )\n",
      "<other> <- ’\n",
      "<gibberish> <- malikkk\n",
      "<gibberish> <- jzzgus\n",
      "<gibberish> <- hjvjg\n",
      "<gibberish> <- fhjkhje\n",
      "<gibberish> <- kgjka\n",
      "<gibberish> <- nnnee\n",
      "<number> <- 17\n",
      "<number> <- 21\n",
      "<gibberish> <- fififififiififififififififiifififa\n",
      "<number> <- 76\n",
      "<other> <- :\n",
      "<gibberish> <- ghujurfhrdzrtz\n",
      "<number> <- 1\n",
      "<gibberish> <- neeehhhaaajjje\n",
      "<number> <- 4\n",
      "<other> <- '\n",
      "<number> <- 0\n",
      "<number> <- 7\n",
      "<gibberish> <- hannnaa\n",
      "<number> <- 4\n",
      "<number> <- 7\n",
      "<other> <- $\n",
      "<number> <- 17\n",
      "<number> <- 17\n",
      "<number> <- 4\n",
      "<number> <- 7\n",
      "<other> <- <\n",
      "<number> <- 23\n",
      "<other> <- \"\n",
      "<other> <- \"\n",
      "<number> <- 23\n",
      "<number> <- 4\n",
      "<number> <- 1\n",
      "<gibberish> <- aaallliiinnnaa\n",
      "<number> <- 23\n",
      "<number> <- 14\n",
      "<other> <- ¨\n",
      "<gibberish> <- ydhrshzrhzr\n",
      "<number> <- 5\n",
      "<number> <- 45\n",
      "<gibberish> <- jjjaa\n",
      "<number> <- 3\n",
      "<number> <- 3\n",
      "<gibberish> <- nnnniiijjjaa\n",
      "<number> <- 6\n",
      "<number> <- 6\n",
      "<gibberish> <- ujhguihoihje\n",
      "<number> <- 8\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<other> <- (\n",
      "<other> <- )\n",
      "<number> <- 5\n",
      "<number> <- 6\n",
      "<number> <- 2\n",
      "<number> <- 5\n",
      "<gibberish> <- xhyhje\n",
      "<other> <- <\n",
      "<gibberish> <- gyxyhdbjy\n",
      "<other> <- <\n",
      "<gibberish> <- skklchasnjbhca\n",
      "<gibberish> <- ahyahjhashahxvztevgxbvafvgsbsbbgaxgchhvxxs\n",
      "<gibberish> <- luhfznjgfghzrghffgtfhjgftfg\n",
      "<gibberish> <- fffzfffzfzfftuftutzguhjhjjkljkcjlcklgzghkckkcsdklkllwehahklwhjkdfklxckuiodhjchjnmddnsmasnmsszusjksdzasdznkasduidsnsjkjhhjffhfhjfzusbjkfjkfzklcuilczovhuildfuduiofje\n",
      "<gibberish> <- hahahahahahahahahahahahahahahahahahahahahahahahahahahahahh\n",
      "<gibberish> <- wtfff\n",
      "<other> <- +\n",
      "<gibberish> <- dfgfhhn\n",
      "<other> <- +\n",
      "<gibberish> <- dfghhn\n",
      "<number> <- 9\n",
      "<number> <- 2\n",
      "<other> <- +\n",
      "<number> <- 20\n",
      "<number> <- 4\n",
      "<gibberish> <- hdgdxghujhguzdcdcdcdcdcdcdcdcdcdcdcfgbuzjhujzhnguuz\n",
      "<gibberish> <- hejhej\n",
      "<gibberish> <- clplojjugzjhugzuftrsrestutgzuhjknhzda\n",
      "<gibberish> <- ztfuhiohnjhf\n",
      "<number> <- 5\n",
      "<gibberish> <- jmbighjdfokl\n",
      "<gibberish> <- kyxpovfwe\n",
      "<number> <- 8\n",
      "<number> <- 9\n",
      "<gibberish> <- zhvuxcvnjba\n",
      "<gibberish> <- llxkgjdfiughfubjcvnkmxcigjrisdpfopdhit\n",
      "<number> <- 9\n",
      "<gibberish> <- hihihihiohihihihihihhihihihihihihihihihihih\n",
      "<other> <- :)\n",
      "<gibberish> <- hahahahahhahahahahahahhahahahhahahahhahahahaha\n",
      "<other> <- :)\n",
      "<number> <- 10\n",
      "<other> <- :)\n",
      "<other> <- :)\n",
      "<gibberish> <- juhhhuu\n",
      "<other> <- :)\n",
      "<number> <- 5\n",
      "<other> <- :)\n",
      "<number> <- 20\n",
      "<number> <- 3\n",
      "<gibberish> <- ghwhwufvjgdgtehiogjzbio\n",
      "<number> <- 6\n",
      "<number> <- 6\n",
      "<number> <- 5\n",
      "<number> <- 9\n",
      "<other> <- =)\n",
      "<number> <- 12\n",
      "<other> <- :(\n",
      "<number> <- 4\n",
      "<number> <- 1\n"
     ]
    }
   ],
   "source": [
    "conversations_gr = [[group_tokens(token) for token in message] for message in conversations]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T14:26:06.745802Z",
     "start_time": "2020-04-25T14:26:06.721864Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['iti', 'pogovarjati'],\n",
       " ['kip'],\n",
       " ['kdo', 'tar'],\n",
       " ['kaj', 'kip'],\n",
       " ['iti', 'vena'],\n",
       " ['kam'],\n",
       " ['park'],\n",
       " ['zadusiti', 'drug'],\n",
       " ['park', 'kodeljev'],\n",
       " ['kdo', 'pis', 'moci'],\n",
       " ['gregr'],\n",
       " ['gregor'],\n",
       " ['behinda'],\n",
       " ['neh'],\n",
       " ['bols', 'poves', 'kdo'],\n",
       " ['tar'],\n",
       " ['<gibberish>'],\n",
       " ['ok'],\n",
       " ['<gibberish>'],\n",
       " ['reklo', 'pogledati', 'ta', 'zbezati'],\n",
       " ['<gibberish>'],\n",
       " ['zakaj', 'pises'],\n",
       " ['kdo', 'mumada', '<number>'],\n",
       " ['ubiti'],\n",
       " ['magoca', 'postati', 'prevec', 'nasilen', '?', '?'],\n",
       " ['ja', 'nasilen'],\n",
       " ['napa', 'is', 'kaj'],\n",
       " ['kaj'],\n",
       " ['upati', 'nov', 'naslednji', 'kniga', 'stup', 'cav'],\n",
       " ['hus', '<gibberish>'],\n",
       " [],\n",
       " ['gregor'],\n",
       " ['ohc',\n",
       "  '<number>',\n",
       "  '.',\n",
       "  'hizti',\n",
       "  '.',\n",
       "  'zzz',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  'z',\n",
       "  'rt',\n",
       "  'fg',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'daws',\n",
       "  'aadw',\n",
       "  'a',\n",
       "  '<gibberish>',\n",
       "  'fff',\n",
       "  'w',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'jgdt',\n",
       "  ',',\n",
       "  'ttv',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'weu',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'utilo'],\n",
       " ['ja'],\n",
       " ['neh'],\n",
       " ['gn', '<gibberish>', '<gibberish>', ','],\n",
       " ['asje',\n",
       "  ',',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  'iwr',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '.',\n",
       "  '<gibberish>'],\n",
       " ['zakaj', 'pogovor'],\n",
       " ['fortniti'],\n",
       " ['nev'],\n",
       " ['nev'],\n",
       " ['kaj'],\n",
       " ['splotches', 'kdo'],\n",
       " ['<other>', '<other>', '<other>'],\n",
       " ['neh'],\n",
       " ['<gibberish>', '<other>', '<other>', '<other>'],\n",
       " ['?', '?'],\n",
       " ['tar', 'nen'],\n",
       " ['napacn'],\n",
       " ['vedeti'],\n",
       " ['kaj', 'misliti'],\n",
       " ['nev'],\n",
       " ['odgovor'],\n",
       " ['boljzanimiv'],\n",
       " ['kaj', 'mislisa', 'ta'],\n",
       " [],\n",
       " ['kak', 'mislisa'],\n",
       " ['kdo'],\n",
       " ['jst'],\n",
       " ['natalio', 'vedeti'],\n",
       " ['pisateljica', 'met', 'domisliti', '?', '?'],\n",
       " ['kdo'],\n",
       " ['.'],\n",
       " ['kaj', 'pomen', 'dst', 'nesti'],\n",
       " ['kdo'],\n",
       " ['ja'],\n",
       " ['tar'],\n",
       " ['ups', '.', 'misliti', 'kdo', 'jst', '.'],\n",
       " ['vtf'],\n",
       " ['kaj'],\n",
       " ['jurje', 'pis', 'kletvica'],\n",
       " ['natalija'],\n",
       " ['kaj', 'natalija'],\n",
       " ['grega'],\n",
       " ['nev'],\n",
       " ['kaj', 'nevo'],\n",
       " [],\n",
       " ['kdo', 'mum', '<number>'],\n",
       " ['kdo'],\n",
       " ['tar'],\n",
       " ['gregor'],\n",
       " ['natalija'],\n",
       " ['midlifa', 'imaga', 'planet', 'custvo', '.'],\n",
       " ['ta', 'hotel', 'pokazati', 'prijateljstvo', '.', '?', '?'],\n",
       " [],\n",
       " ['kaksen', 'odgovor'],\n",
       " ['usec', 'knjiga'],\n",
       " ['smiseln', '.'],\n",
       " ['ja', '.'],\n",
       " ['kdo'],\n",
       " ['kdo', 'mumada', '<number>'],\n",
       " ['natalio'],\n",
       " ['mio', '.'],\n",
       " ['povedati'],\n",
       " ['kaj', 'ps', 'zgodba'],\n",
       " ['odgovoriti'],\n",
       " ['kakati'],\n",
       " ['za'],\n",
       " ['<gibberish>'],\n",
       " ['nev']]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversations_gr[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:22.323468Z",
     "start_time": "2020-04-25T16:07:22.293158Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "class TokenDictionary():\n",
    "    def __init__(self, documents, dict_size=512):\n",
    "        self.dict_size = dict_size\n",
    "        \n",
    "        all_tokens = [token for document in documents for token in document]\n",
    "\n",
    "        # Find most common tokens and construct a dict from them\n",
    "        cnt = Counter(all_tokens)\n",
    "        most_common = cnt.most_common(dict_size-1)\n",
    "\n",
    "        # Out-of-dict token\n",
    "        remaining = len(all_tokens) - sum(count for _, count in most_common)\n",
    "        most_common.append(('<OOD>', remaining))\n",
    "\n",
    "        token_dictionary = [token for token, count in most_common]\n",
    "        self.token_map = {token: i for i, token in enumerate(token_dictionary)}\n",
    "        \n",
    "        # Compute idf for each word\n",
    "        bow = np.stack([self.bag_of_words(document) for document in documents])\n",
    "        num_documents = len(documents)\n",
    "        word_occurences = np.sum(bow>0, axis=0)\n",
    "        \n",
    "        self.idf = np.log(num_documents / word_occurences)\n",
    "        \n",
    "    def get_token(self, token):\n",
    "        \"\"\" Get in-dict token for a given token. \"\"\"\n",
    "        \n",
    "        if token not in self.token_map:\n",
    "            return '<OOD>'\n",
    "        \n",
    "        return token\n",
    "\n",
    "    def bag_of_words(self, tokens, tf_idf=False):\n",
    "        \"\"\" Convert a list of tokens to a bag-of-words representation. \"\"\"\n",
    "        \n",
    "        bow = np.zeros(self.dict_size)\n",
    "        if len(tokens) == 0:\n",
    "            return bow\n",
    "        \n",
    "        for token in tokens:\n",
    "            token = self.get_token(token)\n",
    "            i = self.token_map[token]\n",
    "            bow[i] += 1\n",
    "            \n",
    "        if tf_idf:\n",
    "            tf = np.log(1 + bow / len(tokens))\n",
    "            idf = self.idf\n",
    "            \n",
    "            bow = tf * idf\n",
    "            \n",
    "        return bow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:22.934562Z",
     "start_time": "2020-04-25T16:07:22.871328Z"
    }
   },
   "outputs": [],
   "source": [
    "documents = [message for message in conversations_gr]\n",
    "token_dict = TokenDictionary(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:23.228424Z",
     "start_time": "2020-04-25T16:07:23.201775Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['iti', 'pogovarjati'],\n",
       " ['kip'],\n",
       " ['kdo', 'tar'],\n",
       " ['kaj', 'kip'],\n",
       " ['iti', 'vena'],\n",
       " ['kam'],\n",
       " ['park'],\n",
       " ['zadusiti', 'drug'],\n",
       " ['park', 'kodeljev'],\n",
       " ['kdo', 'pis', 'moci'],\n",
       " ['gregr'],\n",
       " ['gregor'],\n",
       " ['behinda'],\n",
       " ['neh'],\n",
       " ['bols', 'poves', 'kdo'],\n",
       " ['tar'],\n",
       " ['<gibberish>'],\n",
       " ['ok'],\n",
       " ['<gibberish>'],\n",
       " ['reklo', 'pogledati', 'ta', 'zbezati'],\n",
       " ['<gibberish>'],\n",
       " ['zakaj', 'pises'],\n",
       " ['kdo', 'mumada', '<number>'],\n",
       " ['ubiti'],\n",
       " ['magoca', 'postati', 'prevec', 'nasilen', '?', '?'],\n",
       " ['ja', 'nasilen'],\n",
       " ['napa', 'is', 'kaj'],\n",
       " ['kaj'],\n",
       " ['upati', 'nov', 'naslednji', 'kniga', 'stup', 'cav'],\n",
       " ['hus', '<gibberish>'],\n",
       " [],\n",
       " ['gregor'],\n",
       " ['ohc',\n",
       "  '<number>',\n",
       "  '.',\n",
       "  'hizti',\n",
       "  '.',\n",
       "  'zzz',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  'z',\n",
       "  'rt',\n",
       "  'fg',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'daws',\n",
       "  'aadw',\n",
       "  'a',\n",
       "  '<gibberish>',\n",
       "  'fff',\n",
       "  'w',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'jgdt',\n",
       "  ',',\n",
       "  'ttv',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'weu',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  'utilo'],\n",
       " ['ja'],\n",
       " ['neh'],\n",
       " ['gn', '<gibberish>', '<gibberish>', ','],\n",
       " ['asje',\n",
       "  ',',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  'iwr',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '.',\n",
       "  '<gibberish>'],\n",
       " ['zakaj', 'pogovor'],\n",
       " ['fortniti'],\n",
       " ['nev'],\n",
       " ['nev'],\n",
       " ['kaj'],\n",
       " ['splotches', 'kdo'],\n",
       " ['<other>', '<other>', '<other>'],\n",
       " ['neh'],\n",
       " ['<gibberish>', '<other>', '<other>', '<other>'],\n",
       " ['?', '?'],\n",
       " ['tar', 'nen'],\n",
       " ['napacn'],\n",
       " ['vedeti'],\n",
       " ['kaj', 'misliti'],\n",
       " ['nev'],\n",
       " ['odgovor'],\n",
       " ['boljzanimiv'],\n",
       " ['kaj', 'mislisa', 'ta'],\n",
       " [],\n",
       " ['kak', 'mislisa'],\n",
       " ['kdo'],\n",
       " ['jst'],\n",
       " ['natalio', 'vedeti'],\n",
       " ['pisateljica', 'met', 'domisliti', '?', '?'],\n",
       " ['kdo'],\n",
       " ['.'],\n",
       " ['kaj', 'pomen', 'dst', 'nesti'],\n",
       " ['kdo'],\n",
       " ['ja'],\n",
       " ['tar'],\n",
       " ['ups', '.', 'misliti', 'kdo', 'jst', '.'],\n",
       " ['vtf'],\n",
       " ['kaj'],\n",
       " ['jurje', 'pis', 'kletvica'],\n",
       " ['natalija'],\n",
       " ['kaj', 'natalija'],\n",
       " ['grega'],\n",
       " ['nev'],\n",
       " ['kaj', 'nevo'],\n",
       " [],\n",
       " ['kdo', 'mum', '<number>'],\n",
       " ['kdo'],\n",
       " ['tar'],\n",
       " ['gregor'],\n",
       " ['natalija'],\n",
       " ['midlifa', 'imaga', 'planet', 'custvo', '.'],\n",
       " ['ta', 'hotel', 'pokazati', 'prijateljstvo', '.', '?', '?'],\n",
       " [],\n",
       " ['kaksen', 'odgovor'],\n",
       " ['usec', 'knjiga'],\n",
       " ['smiseln', '.'],\n",
       " ['ja', '.'],\n",
       " ['kdo'],\n",
       " ['kdo', 'mumada', '<number>'],\n",
       " ['natalio'],\n",
       " ['mio', '.'],\n",
       " ['povedati'],\n",
       " ['kaj', 'ps', 'zgodba'],\n",
       " ['odgovoriti'],\n",
       " ['kakati'],\n",
       " ['za'],\n",
       " ['<gibberish>'],\n",
       " ['nev']]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:23.544483Z",
     "start_time": "2020-04-25T16:07:23.523494Z"
    }
   },
   "outputs": [],
   "source": [
    "conversations_in_dict = [[token_dict.get_token(token) for token in message] for message in conversations_gr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:24.789466Z",
     "start_time": "2020-04-25T16:07:24.760131Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['iti', 'pogovarjati'],\n",
       " ['kip'],\n",
       " ['kdo', 'tar'],\n",
       " ['kaj', 'kip'],\n",
       " ['iti', 'vena'],\n",
       " ['kam'],\n",
       " ['park'],\n",
       " ['zadusiti', 'drug'],\n",
       " ['park', '<OOD>'],\n",
       " ['kdo', 'pis', 'moci'],\n",
       " ['<OOD>'],\n",
       " ['gregor'],\n",
       " ['<OOD>'],\n",
       " ['neh'],\n",
       " ['<OOD>', '<OOD>', 'kdo'],\n",
       " ['tar'],\n",
       " ['<gibberish>'],\n",
       " ['ok'],\n",
       " ['<gibberish>'],\n",
       " ['reklo', 'pogledati', 'ta', 'zbezati'],\n",
       " ['<gibberish>'],\n",
       " ['zakaj', 'pises'],\n",
       " ['kdo', 'mumada', '<number>'],\n",
       " ['ubiti'],\n",
       " ['<OOD>', 'postati', 'prevec', 'nasilen', '?', '?'],\n",
       " ['ja', 'nasilen'],\n",
       " ['<OOD>', 'is', 'kaj'],\n",
       " ['kaj'],\n",
       " ['upati', 'nov', '<OOD>', '<OOD>', '<OOD>', 'cav'],\n",
       " ['<OOD>', '<gibberish>'],\n",
       " [],\n",
       " ['gregor'],\n",
       " ['<OOD>',\n",
       "  '<number>',\n",
       "  '.',\n",
       "  '<OOD>',\n",
       "  '.',\n",
       "  '<OOD>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  '<OOD>',\n",
       "  'rt',\n",
       "  '<OOD>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<OOD>',\n",
       "  '<OOD>',\n",
       "  'a',\n",
       "  '<gibberish>',\n",
       "  '<OOD>',\n",
       "  'w',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<OOD>',\n",
       "  ',',\n",
       "  '<OOD>',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<OOD>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<gibberish>',\n",
       "  '<OOD>'],\n",
       " ['ja'],\n",
       " ['neh'],\n",
       " ['<OOD>', '<gibberish>', '<gibberish>', ','],\n",
       " ['<OOD>',\n",
       "  ',',\n",
       "  '<gibberish>',\n",
       "  '<number>',\n",
       "  '<OOD>',\n",
       "  '<number>',\n",
       "  '<gibberish>',\n",
       "  '.',\n",
       "  '<gibberish>'],\n",
       " ['zakaj', 'pogovor'],\n",
       " ['fortniti'],\n",
       " ['nev'],\n",
       " ['nev'],\n",
       " ['kaj'],\n",
       " ['<OOD>', 'kdo'],\n",
       " ['<other>', '<other>', '<other>'],\n",
       " ['neh'],\n",
       " ['<gibberish>', '<other>', '<other>', '<other>'],\n",
       " ['?', '?'],\n",
       " ['tar', '<OOD>'],\n",
       " ['<OOD>'],\n",
       " ['vedeti'],\n",
       " ['kaj', 'misliti'],\n",
       " ['nev'],\n",
       " ['odgovor'],\n",
       " ['<OOD>'],\n",
       " ['kaj', 'mislisa', 'ta'],\n",
       " [],\n",
       " ['kak', 'mislisa'],\n",
       " ['kdo'],\n",
       " ['jst'],\n",
       " ['natalio', 'vedeti'],\n",
       " ['<OOD>', '<OOD>', '<OOD>', '?', '?'],\n",
       " ['kdo'],\n",
       " ['.'],\n",
       " ['kaj', 'pomen', '<OOD>', '<OOD>'],\n",
       " ['kdo'],\n",
       " ['ja'],\n",
       " ['tar'],\n",
       " ['ups', '.', 'misliti', 'kdo', 'jst', '.'],\n",
       " ['vtf'],\n",
       " ['kaj'],\n",
       " ['jurje', 'pis', '<OOD>'],\n",
       " ['natalija'],\n",
       " ['kaj', 'natalija'],\n",
       " ['<OOD>'],\n",
       " ['nev'],\n",
       " ['kaj', '<OOD>'],\n",
       " [],\n",
       " ['kdo', 'mum', '<number>'],\n",
       " ['kdo'],\n",
       " ['tar'],\n",
       " ['gregor'],\n",
       " ['natalija'],\n",
       " ['<OOD>', '<OOD>', 'planet', 'custvo', '.'],\n",
       " ['ta', 'hotel', 'pokazati', '<OOD>', '.', '?', '?'],\n",
       " [],\n",
       " ['kaksen', 'odgovor'],\n",
       " ['usec', 'knjiga'],\n",
       " ['<OOD>', '.'],\n",
       " ['ja', '.'],\n",
       " ['kdo'],\n",
       " ['kdo', 'mumada', '<number>'],\n",
       " ['natalio'],\n",
       " ['mio', '.'],\n",
       " ['povedati'],\n",
       " ['kaj', '<OOD>', 'zgodba'],\n",
       " ['odgovoriti'],\n",
       " ['kakati'],\n",
       " ['za'],\n",
       " ['<gibberish>'],\n",
       " ['nev']]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversations_in_dict[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert messages to a bag-of-words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:25.507578Z",
     "start_time": "2020-04-25T16:07:25.421848Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 1.],\n",
       "       [0., 0., 0., ..., 0., 0., 1.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_bow = np.stack([token_dict.bag_of_words(message) for message in conversations_gr])\n",
    "conv_bow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:27.070420Z",
     "start_time": "2020-04-25T16:07:26.962869Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.95775314, 5.12802351, 3.43892895, ..., 0.66019099, 0.66019099,\n",
       "       2.14000525])"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_bow = np.stack([token_dict.bag_of_words(message, tf_idf=True) for message in conversations_gr])\n",
    "conv_bow.sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:09:21.950437Z",
     "start_time": "2020-04-25T16:09:21.922840Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.46279805, 0.39755665, 0.        , 0.51761962, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 1.07462089, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.17365295])"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_bow[1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-25T16:07:28.261165Z",
     "start_time": "2020-04-25T16:07:28.247507Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.128023513834108"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_bow.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "notify_time": "30",
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "103px",
    "width": "220px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "501.85px",
    "left": "1550px",
    "right": "20px",
    "top": "120px",
    "width": "350px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}