Final_build_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "import re\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "import gensim\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dense, Dropout, Embedding, Flatten, Conv1D, MaxPooling1D, LSTM\n",
    "from keras import utils\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_columns = [\"target\", \"ids\", \"date\", \"flag\", \"user\", \"text\"]\n",
    "data = pd.read_csv('/Users/steven/Desktop/PycharmProjects/SMDA/project/tweet_sentiment.csv', encoding='ISO-8859-1',names=data_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>ids</th>\n",
       "      <th>date</th>\n",
       "      <th>flag</th>\n",
       "      <th>user</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>1467810917</td>\n",
       "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>mattycus</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>1467811184</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>ElleCTF</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>1467811193</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Karoli</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     target         ids                          date      flag  \\\n",
       "0  NEGATIVE  1467810369  Mon Apr 06 22:19:45 PDT 2009  NO_QUERY   \n",
       "1  NEGATIVE  1467810672  Mon Apr 06 22:19:49 PDT 2009  NO_QUERY   \n",
       "2  NEGATIVE  1467810917  Mon Apr 06 22:19:53 PDT 2009  NO_QUERY   \n",
       "3  NEGATIVE  1467811184  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY   \n",
       "4  NEGATIVE  1467811193  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY   \n",
       "\n",
       "              user                                               text  \n",
       "0  _TheSpecialOne_  @switchfoot http://twitpic.com/2y1zl - Awww, t...  \n",
       "1    scotthamilton  is upset that he can't update his Facebook by ...  \n",
       "2         mattycus  @Kenichan I dived many times for the ball. Man...  \n",
       "3          ElleCTF    my whole body feels itchy and like its on fire   \n",
       "4           Karoli  @nationwideclass no, it's not behaving at all....  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decode_map = {0: \"NEGATIVE\", 2: \"NEUTRAL\", 4: \"POSITIVE\"}\n",
    "\n",
    "def decode_sentiment(label):\n",
    "    return decode_map[int(label)]\n",
    "\n",
    "data.target = data.target.apply(lambda x: decode_sentiment(x))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7cAAAHiCAYAAAAzuDtuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de/yuZV0n+s9XyFOKoKzYCiiWqwPa1nQFNNZMSSHYAadtJuMOcrNlN+J0nEmq2RtHM3Wasnhl7heTJJSJaAeoMCLUyinMpbIlJIclooAHliwOmWf97j+eay0ffv5+v/VbHFzrwvf79Xpez31/r+u+r+t50Nfz+qz7vq9fdXcAAABgZvfZ2xMAAACAu0q4BQAAYHrCLQAAANMTbgEAAJiecAsAAMD0hFsAAACmJ9wCwN2kql5YVb+/wb6vqapfvpPj3OljV5znrVX1f47tZ1fVX97Vcy6d+6qq+u6xveHvZZXzfFdVvW+d9kdW1Seqar87OVUA7iWEWwD2qqq6rqo+VVX/XFW3VtXfVdVPVNWGfqOq6oiq6qra/x6e51dknL2lu1/b3cftrt9Gg3V3P7a733o3zOtvu/ublsa/rqq+d6n9Q939oO7+wl0dC4C5CbcA7At+sLsfnORRSV6W5AVJXr13p8SdcW8N/wDs+4RbAPYZ3X1bd1+U5EeTnFJVj0uSqvr+qnp3Vd1eVddX1QuXDvub8X7ruD31O6rqG6rqzVV1c1V9vKpeW1UH7jygql5QVTeOq8Xvq6pjR/0+VXVGVb1/HHtBVT10rXF293mq6g1V9dGquq2q/qaqHruiy8FVdemYx19X1aOWjv3m0bZjzPGZa4xxcFX92bjqvaOq/natq95V9X1V9U9jPr+VpJbafryq3ja2q6peUVU3je/8yqp6XFWdluTZSX5+fAd/OvpfN77T9yT5l6raf+UV1iT3r6rXj8/6rqp6/NLYXVWPWdrfdXW4qr67qm4Y27+X5JFJ/nSM//Mrr6hX1SOq6qLxXWyrqucunfeF47/peWMeV1XVlrX/CwIwE+EWgH1Od/9DkhuSfNco/UuSk5McmOT7k/z7qnr6aPvX4/3AcXvq32cR2l6a5BFJviXJ4UlemCRV9U1Jnp/k28fV4qcmuW6c4z8keXqSfzOOvSXJK9cZZ3felGRzkq9L8q4kr13R/uwkL05ycJIrdrZX1dcmuTTJH4xjn5Xkt6vqyFXG+LksvqtNSQ5J8otJemWnqjo4yR8l+c9jvPcnefIa8z4ui8/7jUkekuSZSW7u7rPHHP/r+A5+cOmYk7L4b3Ngd39+lXOemOQNSR46PtefVNXXrDH+qrr7x5J8KIsr/Q/q7v+6Srfzs/g+HpHkGUl+paqestT+Q6PPgUkuSvJbezIHAPZdwi0A+6oPZxGE0t1v7e4ru/uL3f2eJK/LIoCuqru3dfel3f2Z7t6e5NeX+n8hyf2SHFlVX9Pd13X3+0fbTyT5pe6+obs/k0UgfsadvdW2u8/p7n9eOtfjq+ohS13+vLv/ZrT/UpLvqKrDk/xAkuu6+3e7+/Pd/e4kf5jkR1YZ5nNJHp7kUd39ufGM6peF2yRPS3JVd7+xuz+X5DeSfHSNqX8uyYOTfHOS6u6ru/sju/m4Z3X39d39qTXa37k09q8nuX+SY3Zzzj0yvrsnJ3lBd3+6u69I8jtZ/MPITm/r7ovHM7q/l+Txq5wKgAkJtwDsqw5NsiNJquroqnpLVW2vqtuyCKEHr3VgVR1SVeePW49vT/L7O/t397YkP51F2Lxp9HvEOPRRSf543OJ7a5KrswjDh+zp5Ktqv6p62bjF+fZ86erw8ryv37nR3Z8Yn/cRYx5H75zHmMuzk/wvqwz1q0m2JfnLqrq2qs5YY0qPWDFeL+8v6+43Z3FF85VZfEdnV9UBu/nIq55rtfbu/mK+dHX17vSIJDu6+5+Xah/M4n9LOy0H+k9mcbu054QB7gWEWwD2OVX17VkEkreN0h9kcQvp4d39kCT/b770vOhqVyl/ZdS/tbsPSPK/L/VPd/9Bd39nFiGyk7x8NF2f5ITuPnDpdf/uvnGNcdbz77K4Ffd7s7i194idH2+pz+FLn/lBWVyp/vCYx1+vmMeDuvvfrxxkXBn+ue7++ixuuf3Znc8Qr/CRFePV8v4q5z2ru5+U5Mgsbk/+Tzub1jpkrXMNy2PfJ8lhWXzWZBEyH7jUd7UQv5FxPpzkoVX14KXaI5PcuJu5AXAvINwCsM+oqgOq6geyeCby97v7ytH04CyuyH26qo7KIjjutD3JF5N8/VLtwUk+keS2qjo0XwpmqapvqqqnVNX9knw6yafG8ckiNL9k58JOVbWpqk5cZ5z1PDjJZ5LcnEVw+5VV+jytqr6zqu6bxbO3l3f39Un+LMk3VtWPVdXXjNe3V9W3rDxBVf1AVT1mhNXbsrjS/MWV/ZL8eZLHVtUPjyuVP5k1QuQY6+jxTOy/ZPE97Tznx/bgO1j2pKWxfzqL7+by0XZFkn83rnYfn3VuOV9v/PHd/V2Sl1bV/avqf01yahZX7gG4lxNuAdgX/GlV/XMWVyx/KYtnMp+z1P68JC8aff6fJBfsbOjuTyZ5SZL/MW7hPSbJf0nyxCzC3p9nsZDSTvfL4s8NfTyLW1S/LskvjLbfzOIK8V+OsS5PcvQ646znvCxuib0xyXvzpSC37A+SnJnF7chPyuIKc8ZttcdlsZDUh8c8Xz7mvtLmJH+VRZj/+yS/3d1vWdmpuz+exTO7L8sicG9O8j/WmPsBSf57FgtqfXD0/9XR9uosnle+tar+ZM1P/+UuzGIV7FuS/FiSHx7P3ybJTyX5wSQ7b79e77wvTfKfx/j/cZX2k7K4Sv7hJH+c5Mzu/qs9mCcAk6rV15wAAACAebhyCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMb/+9PYG728EHH9xHHHHE3p4GAAAA94B3vvOdH+/uTSvr97pwe8QRR2Tr1q17exoAAADcA6rqg6vV3ZYMAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADT21C4raqfqaqrquofq+p1VXX/qnp0Vb29qrZV1eur6r6j7/3G/rbRfsTSeX5h1N9XVU9dqh8/atuq6oyl+qpjAAAAwLLdhtuqOjTJTybZ0t2PS7JfkmcleXmSV3T3Y5LckuTUccipSW4Z9VeMfqmqI8dxj01yfJLfrqr9qmq/JK9MckKSI5OcNPpmnTEAAABgl43elrx/kgdU1f5JHpjkI0mekuSNo/3cJE8f2yeO/Yz2Y6uqRv387v5Md38gybYkR43Xtu6+trs/m+T8JCeOY9YaAwAAAHbZbbjt7huT/LckH8oi1N6W5J1Jbu3uz49uNyQ5dGwfmuT6ceznR/+HLddXHLNW/WHrjAEAAAC77L+7DlV1UBZXXR+d5NYkb8jituJ9RlWdluS0JHnkIx+5l2ezviPO+PO9PQUA1nDdy75/b0/hq4LfQoB918y/hRu5Lfl7k3ygu7d39+eS/FGSJyc5cNymnCSHJblxbN+Y5PAkGe0PSXLzcn3FMWvVb15njDvo7rO7e0t3b9m0adMGPhIAAAD3JhsJtx9KckxVPXA8B3tskvcmeUuSZ4w+pyS5cGxfNPYz2t/c3T3qzxqrKT86yeYk/5DkHUk2j5WR75vFolMXjWPWGgMAAAB22cgzt2/PYlGndyW5chxzdpIXJPnZqtqWxfOxrx6HvDrJw0b9Z5OcMc5zVZILsgjGf5Hk9O7+wnim9vlJLklydZILRt+sMwYAAADssttnbpOku89McuaK8rVZrHS8su+nk/zIGud5SZKXrFK/OMnFq9RXHQMAAACWbfRPAQEAAMA+S7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA09ttuK2qb6qqK5Zet1fVT1fVQ6vq0qq6ZrwfNPpXVZ1VVduq6j1V9cSlc50y+l9TVacs1Z9UVVeOY86qqhr1VccAAACAZbsNt939vu5+Qnc/IcmTknwyyR8nOSPJZd29OcllYz9JTkiyebxOS/KqZBFUk5yZ5OgkRyU5cymsvirJc5eOO37U1xoDAAAAdtnT25KPTfL+7v5gkhOTnDvq5yZ5+tg+Mcl5vXB5kgOr6uFJnprk0u7e0d23JLk0yfGj7YDuvry7O8l5K8612hgAAACwy56G22cled3YPqS7PzK2P5rkkLF9aJLrl465YdTWq9+wSn29Me6gqk6rqq1VtXX79u17+JEAAACY3YbDbVXdN8kPJXnDyrZxxbXvxnl9mfXG6O6zu3tLd2/ZtGnTPTkNAAAA9kF7cuX2hCTv6u6Pjf2PjVuKM95vGvUbkxy+dNxho7Ze/bBV6uuNAQAAALvsSbg9KV+6JTlJLkqyc8XjU5JcuFQ/eayafEyS28atxZckOa6qDhoLSR2X5JLRdntVHTNWST55xblWGwMAAAB22X8jnarqa5N8X5L/a6n8siQXVNWpST6Y5JmjfnGSpyXZlsXKys9Jku7eUVUvTvKO0e9F3b1jbD8vyWuSPCDJm8ZrvTEAAABglw2F2+7+lyQPW1G7OYvVk1f27SSnr3Gec5Kcs0p9a5LHrVJfdQwAAABYtqerJQMAAMA+R7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgehsKt1V1YFW9sar+qaqurqrvqKqHVtWlVXXNeD9o9K2qOquqtlXVe6rqiUvnOWX0v6aqTlmqP6mqrhzHnFVVNeqrjgEAAADLNnrl9jeT/EV3f3OSxye5OskZSS7r7s1JLhv7SXJCks3jdVqSVyWLoJrkzCRHJzkqyZlLYfVVSZ67dNzxo77WGAAAALDLbsNtVT0kyb9O8uok6e7PdvetSU5Mcu7odm6Sp4/tE5Oc1wuXJzmwqh6e5KlJLu3uHd19S5JLkxw/2g7o7su7u5Oct+Jcq40BAAAAu2zkyu2jk2xP8rtV9e6q+p2q+tokh3T3R0afjyY5ZGwfmuT6peNvGLX16jesUs86YwAAAMAuGwm3+yd5YpJXdfe3JfmXrLg9eFxx7bt/ehsbo6pOq6qtVbV1+/bt9+Q0AAAA2AdtJNzekOSG7n772H9jFmH3Y+OW4oz3m0b7jUkOXzr+sFFbr37YKvWsM8YddPfZ3b2lu7ds2rRpAx8JAACAe5Pdhtvu/miS66vqm0bp2CTvTXJRkp0rHp+S5MKxfVGSk8eqycckuW3cWnxJkuOq6qCxkNRxSS4ZbbdX1TFjleSTV5xrtTEAAABgl/032O8/JHltVd03ybVJnpNFML6gqk5N8sEkzxx9L07ytCTbknxy9E1376iqFyd5x+j3ou7eMbafl+Q1SR6Q5E3jlSQvW2MMAAAA2GVD4ba7r0iyZZWmY1fp20lOX+M85yQ5Z5X61iSPW6V+82pjAAAAwLKN/p1bAAAA2GcJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6Gwq3VXVdVV1ZVVdU1dZRe2hVXVpV14z3g0a9quqsqtpWVe+pqicuneeU0f+aqjplqf6kcf5t49habwwAAABYtidXbr+nu5/Q3VvG/hlJLuvuzUkuG/tJckKSzeN1WpJXJYugmuTMJEcnOSrJmUth9VVJnrt03PG7GQMAAAB2uSu3JZ+Y5NyxfW6Spy/Vz+uFy5McWFUPT/LUJJd2947uviXJpUmOH20HdPfl3d1JzltxrtXGAAAAgF02Gm47yV9W1Tur6rRRO6S7PzK2P5rkkLF9aJLrl469YdTWq9+wSn29MQAAAGCX/TfY7zu7+8aq+rokl1bVPy03dndXVd/909vYGCNwn5Ykj3zkI+/JaQAAALAP2tCV2+6+cbzflOSPs3hm9mPjluKM95tG9xuTHL50+GGjtl79sFXqWWeMlfM7u7u3dPeWTZs2beQjAQAAcC+y23BbVV9bVQ/euZ3kuCT/mOSiJDtXPD4lyYVj+6IkJ49Vk49Jctu4tfiSJMdV1UFjIanjklwy2m6vqmPGKsknrzjXamMAAADALhu5LfmQJH88/jrP/kn+oLv/oqrekeSCqjo1yQeTPHP0vzjJ05JsS/LJJM9Jku7eUVUvTvKO0e9F3b1jbD8vyWuSPCDJm8YrSV62xhgAAACwy27DbXdfm+Txq9RvTnLsKvVOcvoa5zonyTmr1LcmedxGxwAAAIBld+VPAQEAAMA+QbgFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA09twuK2q/arq3VX1Z2P/0VX19qraVlWvr6r7jvr9xv620X7E0jl+YdTfV1VPXaofP2rbquqMpfqqYwAAAMCyPbly+1NJrl7af3mSV3T3Y5LckuTUUT81yS2j/orRL1V1ZJJnJXlskuOT/PYIzPsleWWSE5IcmeSk0Xe9MQAAAGCXDYXbqjosyfcn+Z2xX0mekuSNo8u5SZ4+tk8c+xntx47+JyY5v7s/090fSLItyVHjta27r+3uzyY5P8mJuxkDAAAAdtnoldvfSPLzSb449h+W5Nbu/vzYvyHJoWP70CTXJ8lov23031Vfccxa9fXGAAAAgF12G26r6geS3NTd7/wKzOdOqarTqmprVW3dvn373p4OAAAAX2EbuXL75CQ/VFXXZXHL8FOS/GaSA6tq/9HnsCQ3ju0bkxyeJKP9IUluXq6vOGat+s3rjHEH3X12d2/p7i2bNm3awEcCAADg3mS34ba7f6G7D+vuI7JYEOrN3f3sJG9J8ozR7ZQkF47ti8Z+Rvubu7tH/VljNeVHJ9mc5B+SvCPJ5rEy8n3HGBeNY9YaAwAAAHa5K3/n9gVJfraqtmXxfOyrR/3VSR426j+b5Iwk6e6rklyQ5L1J/iLJ6d39hfFM7fOTXJLFaswXjL7rjQEAAAC77L/7Ll/S3W9N8taxfW0WKx2v7PPpJD+yxvEvSfKSVeoXJ7l4lfqqYwAAAMCyu3LlFgAAAPYJwi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9IRbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACY3m7DbVXdv6r+oar+v6q6qqr+y6g/uqreXlXbqur1VXXfUb/f2N822o9YOtcvjPr7quqpS/XjR21bVZ2xVF91DAAAAFi2kSu3n0nylO5+fJInJDm+qo5J8vIkr+juxyS5Jcmpo/+pSW4Z9VeMfqmqI5M8K8ljkxyf5Lerar+q2i/JK5OckOTIJCeNvllnDAAAANhlt+G2Fz4xdr9mvDrJU5K8cdTPTfL0sX3i2M9oP7aqatTP7+7PdPcHkmxLctR4bevua7v7s0nOT3LiOGatMQAAAGCXDT1zO66wXpHkpiSXJnl/klu7+/Ojyw1JDh3bhya5PklG+21JHrZcX3HMWvWHrTPGyvmdVlVbq2rr9u3bN/KRAAAAuBfZULjt7i909xOSHJbFldZvvkdntYe6++zu3tLdWzZt2rS3pwMAAMBX2B6tltzdtyZ5S5LvSHJgVe0/mg5LcuPYvjHJ4Uky2h+S5Obl+opj1qrfvM4YAAAAsMtGVkveVFUHju0HJPm+JFdnEXKfMbqdkuTCsX3R2M9of3N396g/a6ym/Ogkm5P8Q5J3JNk8Vka+bxaLTl00jllrDAAAANhl/913ycOTnDtWNb5Pkgu6+8+q6r1Jzq+qX07y7iSvHv1fneT3qmpbkh1ZhNV091VVdUGS9yb5fJLTu/sLSVJVz09ySZL9kpzT3VeNc71gjTEAAABgl92G2+5+T5JvW6V+bRbP366sfzrJj6xxrpckeckq9YuTXLzRMQAAAGDZHj1zCwAAAPsi4RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMb7fhtqoOr6q3VNV7q+qqqvqpUX9oVV1aVdeM94NGvarqrKraVlXvqaonLp3rlNH/mqo6Zan+pKq6chxzVlXVemMAAADAso1cuf18kp/r7iOTHJPk9Ko6MskZSS7r7s1JLhv7SXJCks3jdVqSVyWLoJrkzCRHJzkqyZlLYfVVSZ67dNzxo77WGAAAALDLbsNtd3+ku981tv85ydVJDk1yYpJzR7dzkzx9bJ+Y5LxeuDzJgVX18CRPTXJpd+/o7luSXJrk+NF2QHdf3t2d5LwV51ptDAAAANhlj565raojknxbkrcnOaS7PzKaPprkkLF9aJLrlw67YdTWq9+wSj3rjLFyXqdV1daq2rp9+/Y9+UgAAADcC2w43FbVg5L8YZKf7u7bl9vGFde+m+d2B+uN0d1nd/eW7t6yadOme3IaAAAA7IM2FG6r6muyCLav7e4/GuWPjVuKM95vGvUbkxy+dPhho7Ze/bBV6uuNAQAAALtsZLXkSvLqJFd3968vNV2UZOeKx6ckuXCpfvJYNfmYJLeNW4svSXJcVR00FpI6Lsklo+32qjpmjHXyinOtNgYAAADssv8G+jw5yY8lubKqrhi1X0zysiQXVNWpST6Y5Jmj7eIkT0uyLcknkzwnSbp7R1W9OMk7Rr8XdfeOsf28JK9J8oAkbxqvrDMGAAAA7LLbcNvdb0tSazQfu0r/TnL6Guc6J8k5q9S3JnncKvWbVxsDAAAAlu3RaskAAACwLxJuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmJ5wCwAAwPSEWwAAAKYn3AIAADA94RYAAIDpCbcAAABMT7gFAABgesItAAAA0xNuAQAAmN5uw21VnVNVN1XVPy7VHlpVl1bVNeP9oFGvqjqrqrZV1Xuq6olLx5wy+l9TVacs1Z9UVVeOY86qqlpvDAAAAFhpI1duX5Pk+BW1M5Jc1t2bk1w29pPkhCSbx+u0JK9KFkE1yZlJjk5yVJIzl8Lqq5I8d+m443czBgAAANzBbsNtd/9Nkh0ryicmOXdsn5vk6Uv183rh8iQHVtXDkzw1yaXdvaO7b0lyaZLjR8jbE9AAAAfvSURBVNsB3X15d3eS81aca7UxAAAA4A7u7DO3h3T3R8b2R5McMrYPTXL9Ur8bRm29+g2r1NcbAwAAAO7gLi8oNa649t0wlzs9RlWdVlVbq2rr9u3b78mpAAAAsA+6s+H2Y+OW4oz3m0b9xiSHL/U7bNTWqx+2Sn29Mb5Md5/d3Vu6e8umTZvu5EcCAABgVnc23F6UZOeKx6ckuXCpfvJYNfmYJLeNW4svSXJcVR00FpI6Lsklo+32qjpmrJJ88opzrTYGAAAA3MH+u+tQVa9L8t1JDq6qG7JY9fhlSS6oqlOTfDDJM0f3i5M8Lcm2JJ9M8pwk6e4dVfXiJO8Y/V7U3TsXqXpeFisyPyDJm8Yr64wBAAAAd7DbcNvdJ63RdOwqfTvJ6Wuc55wk56xS35rkcavUb15tDAAAAFjpLi8oBQAAAHubcAsAAMD0hFsAAACmJ9wCAAAwPeEWAACA6Qm3AAAATE+4BQAAYHrCLQAAANMTbgEAAJiecAsAAMD0hFsAAACmJ9wCAAAwPeEWAACA6Qm3AAAATE+4BQAAYHrCLQAAANMTbgEAAJiecAsAAMD0hFsAAACmJ9wCAAAwPeEWAACA6Qm3AAAATE+4BQAAYHrCLQAAANMTbgEAAJiecAsAAMD0hFsAAACmJ9wCAAAwPeEWAACA6Qm3AAAATE+4BQAAYHrCLQAAANMTbgEAAJiecAsAAMD0hFsAAACmJ9wCAAAwPeEWAACA6Qm3AAAATE+4BQAAYHrCLQAAANMTbgEAAJiecAsAAMD0hFsAAACmt8+H26o6vqreV1XbquqMvT0fAAAA9j37dLitqv2SvDLJCUmOTHJSVR25d2cFAADAvmafDrdJjkqyrbuv7e7PJjk/yYl7eU4AAADsY/b1cHtokuuX9m8YNQAAANhl/709gbtDVZ2W5LSx+4mqet/enA98lTk4ycf39iTg7lAv39szACblt5B7jUl+Cx+1WnFfD7c3Jjl8af+wUbuD7j47ydlfqUkBX1JVW7t7y96eBwDsLX4LYd+wr9+W/I4km6vq0VV13yTPSnLRXp4TAAAA+5h9+sptd3++qp6f5JIk+yU5p7uv2svTAgAAYB+zT4fbJOnui5NcvLfnAazJIwEAfLXzWwj7gOruvT0HAAAAuEv29WduAQAAYLeEW7iXq6quql9b2v+PVfXCsf3Cqrqxqq5Yeh042o6qqrdW1TVV9a6q+vOq+tYV576iqs5f2n/lqL23qj61dM5nVNVrxvuZVfXSFed5QlVdPbavq6orl4496x78egD4KlJVXxi/Lf9YVW+oqgeO+mFVdeH4zXt/Vf3mWMw0VfXAqnrt+G36x6p6W1U9aLR9oqq+dek3a0dVfWBs/1VVHTGOeWBV3VxVB6yYz59U1Y9W1Y9X1fYVv8dHfuW/IZibcAv3fp9J8sNVdfAa7a/o7icsvW6tqkOSXJDkF7t7c3c/MclLk3zDzoOq6luyWOjtu6rqa5Oku0/v7ickeVqS9y+d841L470uyY+umMOzRn2n71k69ifvwmcHgGWfGr8tj0vy2SQ/UVWV5I+S/El3b07yjUkelOQl45ifSvKx7v7WcdypST6384TdfeXO36ws/qrHfxr737vU55NZLJD6b3fWquohSb4zyZ+O0utX/B6/9575CuDeS7iFe7/PZ7HQxc/swTHPT3Jud//dzkJ3v627/2Spz0lJfi/JXyY5caMn7u7/meSWqjp6qfzM3DHcAsA97W+TPCbJU5J8urt/N0m6+wtZ/Gb+H+PK7sOT3LjzoO5+X3d/5k6M97os/jF3p3+b5JIRfIG7gXALXx1emeTZ41+JV/qZpVug3jJqj03yrt2c80eTnJ/Fj/VJezifXT/wVXVMkh3dfc1S+1uW5rQnoRwAdquq9k9yQpIrs/jNe+dye3ffnuRDWYTfc5K8oKr+vqp+uao238lhL0nyxKp62NhfedfSj664LfkBd3Ic+Kol3MJXgfEjfV6S1W7xXb4t+XtWO76q3l5VV1fVb479LUk+3t0fSnJZkm+rqofuwZRen+QZVXWffPmPe3LH25JfsQfnBYD1PKCqrkiyNYvw+urdHdDdVyT5+iS/muShSd4xHs3ZI9392SxuW37GeFTo27IIvDutvC35U3s6Bny12+f/zi1wt/mNLK7G/u4G+l6V5IlJLkyS7j66qp6R5AdG+0lJvrmqrhv7ByT535L8941MpLuvr6oPJPk347jv2OBnAIC74lPj2dhdquq9SZ6xonZAkkcm2ZYk3f2JLJ7L/aOq+mIWa0tcfSfGf12S/ztJJbmwuz+3m/7AHnDlFr5KdPeOLBaJOnUD3V+Z5Mer6l8t1XauKHmfLJ6R/dbuPqK7j8jimds7c2vyK5Jc29037OGxAHB3uSzJA6vq5CSpqv2S/FqS13T3J6vqyVV10Gi7b5Ijk3zwTo711iSbk5wea03A3U64ha8uv5Zk5arJP7PiGZ8juvujWTxT+9Kq2lZVf5fFv2r/VpLvSnJjd3946Rx/k+TIqnr4HszlDVk857Taj/vyM7fn7cE5AWCPdHdnsbjTj1TVNUn+Z5JPJ/nF0eUbkvx1VV2Z5N1Z3NL8h3dyrC8meWOShyX56xXNK5+5/VdffgZgPbX4/zMAAADMy5VbAAAApifcAgAAMD3hFgAAgOkJtwAAAExPuAUAAGB6wi0AAADTE24BAACYnnALAADA9P5/8jXeK/NtoeAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "target_cnt = Counter(data.target)\n",
    "\n",
    "plt.figure(figsize=(16,8))\n",
    "plt.bar(target_cnt.keys(), target_cnt.values())\n",
    "plt.title(\"Dataset labels distribuition\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT_CLEANING_RE = \"@\\S+|https?:\\S+|http?:\\S|[^A-Za-z0-9]+\"\n",
    "stemmer = SnowballStemmer('english')\n",
    "stop_words = stopwords.words(\"english\")\n",
    "\n",
    "def pre_process(text, stem=False):\n",
    "    \n",
    "    text = re.sub(TEXT_CLEANING_RE, ' ', str(text).lower()).strip()\n",
    "    tokens = []\n",
    "    for token in text.split():\n",
    "        if token not in stop_words:\n",
    "            if stem:\n",
    "                tokens.append(stemmer.stem(token))\n",
    "            else:\n",
    "                tokens.append(token)\n",
    "    return ' '.join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.text = data.text.apply(lambda x: pre_process(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         awww bummer shoulda got david carr third day\n",
       "1    upset update facebook texting might cry result...\n",
       "2    dived many times ball managed save 50 rest go ...\n",
       "3                     whole body feels itchy like fire\n",
       "4                                     behaving mad see\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.text.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split Train and Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN size: 1280000\n",
      "TEST size: 320000\n"
     ]
    }
   ],
   "source": [
    "df_train, df_test = train_test_split(data, test_size=0.2, random_state=42)\n",
    "\n",
    "print(\"TRAIN size:\", len(df_train))\n",
    "print(\"TEST size:\", len(df_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word to Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_train = [x.split() for x in df_train.text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "W2V_SIZE = 300\n",
    "w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE)\n",
    "w2v_model.build_vocab(doc_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(267155090, 295270528)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model.train(doc_train, total_examples=len(doc_train), epochs=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('firing', 0.3121594190597534),\n",
       " ('stove', 0.3113313913345337),\n",
       " ('fires', 0.31128209829330444),\n",
       " ('evacuated', 0.3025902807712555),\n",
       " ('drills', 0.3014261722564697),\n",
       " ('grills', 0.296565443277359),\n",
       " ('burning', 0.29130876064300537),\n",
       " ('trumpet', 0.2877410352230072),\n",
       " ('alarms', 0.27961429953575134),\n",
       " ('evacuate', 0.26365378499031067)]"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model.wv.most_similar(\"fire\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words 290419\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(df_train.text)\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "print(\"Total words\", vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Train and Test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = pad_sequences(tokenizer.texts_to_sequences(df_train.text), maxlen=300)\n",
    "x_test = pad_sequences(tokenizer.texts_to_sequences(df_test.text), maxlen=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Label Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['POSITIVE', 'NEGATIVE', 'NEUTRAL']"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = df_train.target.unique().tolist()\n",
    "labels.append('NEUTRAL')\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_train (1280000, 1)\n",
      "y_test (320000, 1)\n"
     ]
    }
   ],
   "source": [
    "encoder = LabelEncoder()\n",
    "\n",
    "encoder.fit(df_train.target.tolist())\n",
    "\n",
    "y_train = encoder.transform(df_train.target.tolist())\n",
    "y_test = encoder.transform(df_test.target.tolist())\n",
    "\n",
    "y_train = y_train.reshape(-1,1)\n",
    "y_test = y_test.reshape(-1,1)\n",
    "\n",
    "print(\"y_train\",y_train.shape)\n",
    "print(\"y_test\",y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1],\n",
       "       [1],\n",
       "       [1],\n",
       "       [0],\n",
       "       [1],\n",
       "       [1],\n",
       "       [1],\n",
       "       [1],\n",
       "       [1],\n",
       "       [1]])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embedding word from w2v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(290419, 300)\n"
     ]
    }
   ],
   "source": [
    "embedding_matrix = np.zeros((vocab_size, W2V_SIZE))\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if word in w2v_model.wv:\n",
    "        embedding_matrix[i] = w2v_model.wv[word]\n",
    "print(embedding_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(vocab_size, \n",
    "                            W2V_SIZE, \n",
    "                            weights=[embedding_matrix], \n",
    "                            input_length=300, \n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 300, 300)          87125700  \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 300, 300)          0         \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 100)               160400    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 87,286,201\n",
      "Trainable params: 160,501\n",
      "Non-trainable params: 87,125,700\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(embedding_layer)\n",
    "model.add(Dropout(0.5))\n",
    "model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=\"adam\",\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [ ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=0),\n",
    "              EarlyStopping(monitor='val_acc', min_delta=1e-4, patience=5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1152000 samples, validate on 128000 samples\n",
      "Epoch 1/4\n",
      "1152000/1152000 [==============================] - 5834s 5ms/step - loss: 0.4943 - accuracy: 0.7570 - val_loss: 0.4604 - val_accuracy: 0.7809\n",
      "Epoch 2/4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/keras/callbacks/callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_acc` which is not available. Available metrics are: val_loss,val_accuracy,loss,accuracy,lr\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1152000/1152000 [==============================] - 5400s 5ms/step - loss: 0.4798 - accuracy: 0.7666 - val_loss: 0.4536 - val_accuracy: 0.7852\n",
      "Epoch 3/4\n",
      "1152000/1152000 [==============================] - 5541s 5ms/step - loss: 0.4738 - accuracy: 0.7708 - val_loss: 0.4517 - val_accuracy: 0.7869\n",
      "Epoch 4/4\n",
      "1152000/1152000 [==============================] - 5429s 5ms/step - loss: 0.4705 - accuracy: 0.7728 - val_loss: 0.4491 - val_accuracy: 0.7878\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=1024,\n",
    "                    epochs=4,\n",
    "                    validation_split=0.1,\n",
    "                    verbose=1,\n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print test data accuray and loss "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "320000/320000 [==============================] - 476s 1ms/step\n",
      "ACCURACY: 0.7890156507492065\n",
      "LOSS: 0.4479480337142944\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x_test, y_test, batch_size=1024)\n",
    "print(\"ACCURACY:\",score[1])\n",
    "print(\"LOSS:\",score[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot training and validation accuracy and loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXwV1f3/8deHsO+7YFhdETeWCCigImLRWqiKCra1aq3VCm61tfVn3arV1rW2flXq3lqoxQ1aUdkq4kpQNkEQFTUQKSD7nvD5/XEm5Ca5SS6Q5Obmvp+Px31wZ+bM3HPuDZ/PzJmZM+buiIhI+qmV7AqIiEhyKAGIiKQpJQARkTSlBCAikqaUAERE0pQSgIhImlICkD3MLMPMNptZp4osm0xmdoiZVfi1zmZ2qpktj5leYmYDEym7D5/1uJnduK/ri5SmdrIrIPvOzDbHTDYEdgD50fTP3P25vdmeu+cDjSu6bDpw98MrYjtmdinwQ3c/OWbbl1bEtkWKUwJIYe6+JwBHe5iXuvvU0sqbWW13z6uKuomUR3+PyacuoBrMzO4ws3+a2Tgz2wT80MyON7P3zGy9meWa2UNmVicqX9vM3My6RNN/j5ZPNrNNZvaumXXd27LR8tPNbKmZbTCzP5vZ22Z2USn1TqSOPzOzZWa2zsweilk3w8weMLO1ZvY5MLSM7+f/mdn4YvMeNrP7o/eXmtniqD2fRXvnpW0rx8xOjt43NLO/RXX7GOhdrOxNZvZ5tN2PzWxYNP9o4C/AwKh7bU3Md3trzPqXR21fa2Yvm1n7RL6bvfmeC+pjZlPN7Fsz+8bMfhXzOb+NvpONZpZtZgfG624zs1kFv3P0fc6MPudb4CYzO9TMZkSfsSb63prFrN85auPqaPmfzKx+VOcjYsq1N7OtZtaqtPZKHO6uVw14AcuBU4vNuwPYCXyPkOwbAMcBfQlHfwcBS4HRUfnagANdoum/A2uALKAO8E/g7/tQti2wCRgeLbsO2AVcVEpbEqnjK0AzoAvwbUHbgdHAx0AHoBUwM/yZx/2cg4DNQKOYbf8PyIqmvxeVMeAUYBtwTLTsVGB5zLZygJOj9/cC/wVaAJ2BRcXKnge0j36TC6I6HBAtuxT4b7F6/h24NXp/WlTHHkB94P+A6Yl8N3v5PTcDVgFXA/WApkCfaNlvgHnAoVEbegAtgUOKf9fArILfOWpbHnAFkEH4ezwMGAzUjf5O3gbujWnPwuj7bBSV7x8tGwvcGfM5vwBeSvb/w1R7Jb0CelXQD1l6ApheznrXA/+K3scL6o/GlB0GLNyHspcAb8UsMyCXUhJAgnXsF7P8ReD66P1MQldYwbIzigelYtt+D7ggen86sKSMsv8Grozel5UAvor9LYCfx5aNs92FwHej9+UlgGeA38csa0o479OhvO9mL7/nHwGzSyn3WUF9i81PJAF8Xk4dRhR8LjAQ+AbIiFOuP/AFYNH0XODsiv5/VdNf6gKq+b6OnTCzbmb2n+iQfiNwO9C6jPW/iXm/lbJP/JZW9sDYenj4H5tT2kYSrGNCnwV8WUZ9Af4BjIreXxBNF9TjTDN7P+qeWE/Y+y7ruyrQvqw6mNlFZjYv6sZYD3RLcLsQ2rdne+6+EVgHZMaUSeg3K+d77kgI9PGUtaw8xf8e25nZ82a2IqrD08XqsNzDBQdFuPvbhKOJAWZ2FNAJ+M8+1iltKQHUfMUvgXyMsMd5iLs3BW4m7JFXplzCHioAZmYUDVjF7U8dcwmBo0B5l6k+D5xqZpmELqp/RHVsAEwA7iJ0zzQH3kiwHt+UVgczOwh4hNAN0ira7icx2y3vktWVhG6lgu01IXQ1rUigXsWV9T1/DRxcynqlLdsS1alhzLx2xcoUb98fCFevHR3V4aJidehsZhml1ONZ4IeEo5Xn3X1HKeWkFEoA6acJsAHYEp1E+1kVfOa/gV5m9j0zq03oV25TSXV8HrjGzDKjE4I3lFXY3b8hdFM8Tej++TRaVI/QL70ayDezMwl91YnW4UYza27hPonRMcsaE4LgakIu/CnhCKDAKqBD7MnYYsYBPzGzY8ysHiFBveXupR5RlaGs73ki0MnMRptZPTNramZ9omWPA3eY2cEW9DCzloTE9w3hYoMMM7uMmGRVRh22ABvMrCOhG6rAu8Ba4PcWTqw3MLP+Mcv/RugyuoCQDGQvKQGkn18APyaclH2McLK2Urn7KuB84H7Cf+iDgY8Ie34VXcdHgGnAAmA2YS++PP8g9Onv6f5x9/XAtcBLhBOpIwiJLBG3EI5ElgOTiQlO7j4f+DPwQVTmcOD9mHWnAJ8Cq8wstiunYP3XCF01L0XrdwJ+kGC9iiv1e3b3DcAQ4BxCUloKnBQtvgd4mfA9bySckK0fde39FLiRcEHAIcXaFs8tQB9CIpoIvBBThzzgTOAIwtHAV4TfoWD5csLvvMPd39nLtguFJ1BEqkx0SL8SGOHubyW7PpK6zOxZwonlW5Ndl1SkG8GkSpjZUMIVN9sIlxHuIuwFi+yT6HzKcODoZNclVakLSKrKAOBzQt/3d4CzdNJO9pWZ3UW4F+H37v5VsuuTqtQFJCKSpnQEICKSplLqHEDr1q29S5cuya6GiEhKmTNnzhp3L3HpdUolgC5dupCdnZ3saoiIpBQzi3tHvLqARETSlBKAiEiaUgIQEUlTSgAiImlKCUBEJE0pAYiIpCklABGRNJVS9wGIiNQ47rBlC2zYUPbruuugZcsK/WglABGRfbV7N2zeXH7wLu21cWN45Zd46mVRtWrBBRcoAYiIVIj8fNi0ad+D94YNYf3yBtSsXRuaNYOmTcO/zZpBly6F7xN5NWoEVvFPblUCEJHUk5cX9pz3J3hv3lz+59StWzJ4H3LI3gXvBg0qJXhXBCUAEalaO3cW7QLZl+C9dWv5n1OvXslg3L793gXv+vUr//tIIiUAEUnc9u37H7y3by//cxo0KBmMO3ZMPHA3bRoSgJRJCUAkXe3YAStXhteKFeG1ciWsWVN68N65s/ztNmpUNBi3bAldu+5d8K5Tp/LbL0oAIjXO7t2wdm1hUC8I7MXfr1lTct169aBNm8Jg3LYtHHpo4sG7SZNw0lNSgn4pkVSydWvZQb3g/a5dRdczC8E8MzN0pfTrF95nZsKBBxa+b9Gi2p6wlIqnBCBSHeTnw6pVpQf1gun160uu26hRYQAfOLBkUM/MhHbt1K0iJSgBiFS2jRvLDuorVsA335S8GSgjIwTuzEw4/HAYNKhoUC8I8k2bJqddkvKUAET21a5dkJtbelAveB/vevPmzQsDeffu8btj2rYNSUCkkigBiBTnDuvWlX8S9X//K3kXaN26IYgfeCAceyyccUbJ7pgDD4SGDZPTNpEYSgCSXrZvLwzgZZ1EjXeteuvWhUG8d++igb3gfevWOokqKUMJQGqG3bth9eryu2PWri25boMGhYG8b9/43THt2+vGIqlxlACk+tuypfSgXjCdmxv/0sd27UIg79oVBgyI3x3TvLn22iUtJZQAzGwo8CcgA3jc3e8utvwBYFA02RBo6+7No2V/BL5LePjMFOBqd3cz+y/QHtgWrXeau/9v/5ojKSUvL1z6WFZ3zIoV4Sqa4po0KQziJ58cvzumXTvdlCRShnL/d5hZBvAwMATIAWab2UR3X1RQxt2vjSk/BugZvT8B6A8cEy2eBZwE/Dea/oG7Z+9/MyQl5OdDdja88QZMmQLvvVdyr7127dDdkpkJRxwBp55asjvmwANDAhCR/ZLI7lEfYJm7fw5gZuOB4cCiUsqPAm6J3jtQH6gLGFAHWLU/FZYU89lnIdhPmQLTpxfeyNSrF1x9NRx8cNHA3rZtePiFiFS6RBJAJvB1zHQO0DdeQTPrDHQFpgO4+7tmNgPIJSSAv7j74phVnjKzfOAF4A73kk9WMLPLgMsAOnXqlEB1JanWrQuBfsqUsKf/xRdhfseOcPbZMGQIDB4cxpsRkaSq6A7SkcAEd88HMLNDgCOADtHyKWY20N3fInT/rDCzJoQE8CPg2eIbdPexwFiArKysch69I1Vu5054993Cvfzs7HBFTuPG4c7V664LQf+ww3SiVaSaSSQBrAA6xkx3iObFMxK4Mmb6LOA9d98MYGaTgeOBt9x9BYC7bzKzfxC6mkokAKlm3GHx4sKA/9//hqt0MjKgTx+46aYQ8Pv21dgzItVcIglgNnComXUlBP6RwAXFC5lZN6AF8G7M7K+An5rZXYQuoJOAB82sNtDc3deYWR3gTGDqfrVEKs+qVTB1agj4U6eGK3MgPBrvwgvhtNPC3n6zZsmtp4jslXITgLvnmdlo4HXCZaBPuvvHZnY7kO3uE6OiI4HxxfrxJwCnAAsIJ4Rfc/dJZtYIeD0K/hmE4P/XCmuV7J9t2+Cttwr38ufNC/Nbtgz990OGhFeXLkmtpojsH4tz3rXaysrK8uxsXTVa4XbvDkG+4MTtrFnhaVF16kD//iHYn3Ya9OypwclEUpCZzXH3rOLzdZdMuvr668I9/GnTwjAKAEcdBT//eQj6J54YxpoXkRpJCSBdbNoUTtgW7OUvWRLmt2sH3/lOCPinnhquxReRtKAEUFPl5cHs2YV7+e+9F+Y1aAAnnQSXXRaC/lFH6fJMkTSlBFBTuJe863bDhhDce/WC668PAf+EE6B+/WTXVkSqASWAVPbtt6H/viDoL18e5nfqBOeeGwL+KaeEMepFRIpRAkglO3fCO+8UvevWPTwTdtCgwr38Qw9Vt46IlEsJoDpzh0WLCk/cvvkmbN0aLsXs2xduvjkE/D59dNetiOw1JYDq5ptvit51u3JlmH/YYXDxxSHgn3yy7roVkf2mBJBsW7cW3nX7xhuwYEGY36pV0btuO3dObj1FpMZRAqhqu3fDRx8V9uPPmhX69uvWDXfd3nVXCPg9e2pcfBGpVEoAVeGrr4redbtmTZh/9NEwenThXbcNGya3niKSVpQAKsPGjeGu24JHHy5dGua3bw9nnFF41227dkmtpoikNyWAipCXBx98UPSu2/z8sEd/0klw+eUh6B95pC7PFJFqQwlgX7jDsmWFJ25nzAh7/WbQuzf86ldh9Mzjj4d69ZJdWxGRuJQAErV2bdG7br/8Mszv0gXOP7/wrttWrZJaTRGRRCkBlGbHjqJ33c6ZU3jX7SmnFO7lH3ywunVEJCUpARRwh48/LjxxO3Nm4V23xx8Pt94a9vKPOw5q62sTkdSX3pEsN7foXbe5uWH+4YfDJZcU3nXbtGlSqykiUhnSKwFs3Rr27Av28hcuDPNbtQqXZZ52Wvi3U6fk1lNEpAqkRwJ45BH417/g7bfDXbf16sGAAfDDH4a9/B49dNetiKSd9EgAb78druIZMybs5Q8YoLtuRSTtpUcCePppnbgVESkmPfo9FPxFREpIKAGY2VAzW2Jmy8zs13GWP2Bmc6PXUjNbH7Psj2b2sZktNrOHzMJF82bW28wWRNvcM19ERKpGuQnAzDKAh4HTge7AKDPrHlvG3a919x7u3gP4M/BitO4JQH/gGOAo4DjgpGi1R4CfAodGr6EV0SAREUlMIkcAfYBl7v65u+8ExgPDyyg/ChgXvXegPlAXqAfUAVaZWXugqbu/5+4OPAt8fx/bICIi+yCRBJAJfB0znRPNK8HMOgNdgekA7v4uMAPIjV6vu/viaP2cBLd5mZllm1n26tWrE6iuiIgkoqJPAo8EJrh7PoCZHQIcAXQgBPhTzGzg3mzQ3ce6e5a7Z7Vp06aCqysikr4SSQArgI4x0x2iefGMpLD7B+As4D133+zum4HJwPHR+h0S3KaIiFSCRBLAbOBQM+tqZnUJQX5i8UJm1g1oAbwbM/sr4CQzq21mdQgngBe7ey6w0cz6RVf/XAi8sp9tERGRvVBuAnD3PGA08DqwGHje3T82s9vNbFhM0ZHA+OikboEJwGfAAmAeMM/dJ0XLfg48DiyLykze38aIiEjirGi8rt6ysrI8Ozs72dUQEUkpZjbH3bOKz0+PO4FFRKQEJQARkTSlBCAikqaUAERE0pQSgIhImlICEBFJU0oAIiJpSglARCRNKQGIiKQpJQARkTSlBCAikqaUAERE0pQSgIhImlICEBFJU0oAIiJpSglARCRNKQGIiKQpJQARkTSlBCAikqaUAERE0pQSgIhImlICEBFJU0oAIiJpKqEEYGZDzWyJmS0zs1/HWf6Amc2NXkvNbH00f1DM/Llmtt3Mvh8te9rMvohZ1qNimyYiImWpXV4BM8sAHgaGADnAbDOb6O6LCsq4+7Ux5ccAPaP5M4Ae0fyWwDLgjZjN/9LdJ1RAO0REZC8lcgTQB1jm7p+7+05gPDC8jPKjgHFx5o8AJrv71r2vpoiIVLREEkAm8HXMdE40rwQz6wx0BabHWTySkonhTjObH3Uh1Stlm5eZWbaZZa9evTqB6oqISCIq+iTwSGCCu+fHzjSz9sDRwOsxs38DdAOOA1oCN8TboLuPdfcsd89q06ZNBVdXRCR9JZIAVgAdY6Y7RPPiibeXD3Ae8JK77yqY4e65HuwAniJ0NYmISBVJJAHMBg41s65mVpcQ5CcWL2Rm3YAWwLtxtlHivEB0VICZGfB9YOHeVV1ERPZHuVcBuXuemY0mdN9kAE+6+8dmdjuQ7e4FyWAkMN7dPXZ9M+tCOIJ4s9imnzOzNoABc4HL96chIiKyd6xYvK7WsrKyPDs7O9nVEBFJKWY2x92zis/XncAiImlKCUBEJE0pAYiIpCklABGRNKUEICKSppQARETSlBKAiEiaUgIQEanGdu+G3FzIzy+/7N4q905gERGpPLt3wzffwPLl4fXll4XvC6Z37IBPP4VDDqnYz1YCEBGpRAV78PGCe8G8nTuLrtOmDXTpAsceC8OHh/fNm1d83ZQARET2Q35+2QH+q69KBvi2bUNQ79kTzjorvC94deoEjRpVTd2VAEREypCfDytXlh3gd+0qus4BB4Rg3rs3nHNOyQDfsGGVNqFUSgAiktby82HFirIDfF5e0XXatQvB/Ljj4NxzSwb4Bg2qtAn7TAlARGq0vLwQ4OMF9+XL4euvSwb49u1DMO/bF84/v2SAr1+/SptQaZQARCSl5eVBTk7ZAb74JZQHHhiC+fHHw6hRRQN8x441J8CXRwlARKq1XbvKDvA5OUUDvFlhgO/fv2hwLwjw9epVdSuqJyUAEUmqXbvCXnpZAX737sLyZpCZGYL5wIHxA3zdulXditSkBCAilWrnzrID/IoVRQN8rVqFAf6kk4oG986dFeArkhKAiOyXggAfL7gXBPjYJ8/WqgUdOoSAPmhQyQDfoYMCfFVRAhCRMu3YUXaAX7myZIDv2DEE9MGD4wf4OnWquhUSjxKAiACwahVMnw4LFxYN8Lm5RQN8RkZhgB8ypGSAz8xUgE8VSgAiaWrLFnjrLZgyBaZOhfnzw/zatQsD/He+Ez/A11bkqBH0M4qkifx8mDOnMOC/807ov69bFwYMgLvuglNPhR49FODTRUI/s5kNBf4EZACPu/vdxZY/AAyKJhsCbd29uZkNAh6IKdoNGOnuL5tZV2A80AqYA/zI3YsNmSQi+8odPvusMOBPnw7r14dlPXrA1VeHgD9gQPUZm0aqVrkJwMwygIeBIUAOMNvMJrr7ooIy7n5tTPkxQM9o/gygRzS/JbAMeCMq+gfgAXcfb2aPAj8BHqmIRomkqzVrYNq0EPCnTAmXXkIYvuCcc0Kf/SmnhOGGRRI5AugDLHP3zwHMbDwwHFhUSvlRwC1x5o8AJrv7VjMz4BTggmjZM8CtKAGI7JVt22DWrMKA/9FHYX6zZiHQ33BD2Ms/5JBwA5VIrEQSQCbwdcx0DtA3XkEz6wx0BabHWTwSuD963wpY7+4FQzDlRJ8Tb5uXAZcBdOrUKYHqitRcu3eHIF8Q8GfNCpdp1qkDJ5wAd9wRAn7v3urHl/JV9J/ISGCCuxcZesnM2gNHA6/v7QbdfSwwFiArK8vLKS5S43zxRWHAnz4d1q4N848+Gn7+89CtM3AgNG6c3HpK6kkkAawAOsZMd4jmxTMSuDLO/POAl9y94LEJa4HmZlY7Ogooa5siaeXbb2HGjMKTt599FuZnZsL3vhf28AcPDmPSi+yPRBLAbODQ6KqdFYQgf0HxQmbWDWgBvBtnG6OA3xRMuLub2QzCeYHxwI+BV/a69iI1wI4d4ZLMgoCfnR2u4GnSJAyVcPXVYS//8MPVjy8Vq9wE4O55Zjaa0H2TATzp7h+b2e1AtrtPjIqOBMa7e5FuGjPrQjiCeLPYpm8AxpvZHcBHwBP70xCRVLF7NyxYUNitM3NmOJmbkQH9+sEtt4SAf9xxuqNWKpcVi9fVWlZWlmdnZye7GiJ77euvC/fwp02D//0vzO/ePXTpnHpqGPmyadPk1lNqJjOb4+5ZxefrOgGRSrBhQ+jHL9jLX7o0zG/XDk47LezhDx4c+vVFkkUJQKQC7NwJ779fuJf/wQdh6IVGjcKe/RVXhL38I49UP75UH0oAIvvAHT7+uHAP/803w+BqtWpBnz5w440h4Pfrp7HtpfpSAhBJ0MqVhQF/6lT45psw/7DD4KKLQsA/+WRo3jyZtRRJnBKASCk2bQp79gVBf1E0+EmbNoUnbk89NYyzI5KKlABEIrt2wezZhQH/vfcgLw8aNIATT4SLLw4nb48+OnT1iKQ6JQBJW+6wZElhwJ8xI+z1m0FWFvzylyHgH3881K+f7NqKVDwlAEkrq1aF6/AL+vFzcsL8gw+GCy4IAX/QIGjZMrn1FKkKSgBSo5X22MOWLcN1+EOGhH78rl2TW0+RZFACkBqltMce1qsXnnx1990h4PfsqX58ESUASWllPfawZ0+45prCxx42aJDcuopUN0oAknJKe+xh584wYkQI+HrsoUj5lACk2ivtsYfNmxc+9nDIkHAiV8MsiCROCUCqnbIee9i/f3js4ZAh4bGHGRnJrq1I6lICkGqhtMceHnMMXHll4WMPGzVKbj1FahIlAEmKvDyYNAlefz3+Yw8Lhks+4IDk1lOkJlMCkCr37rvhYeZz5xY+9rDgah099lCk6igBSJVZsyacsH3ySejQAZ5/Hs46C2rrr1AkKXQrjFS63bth7Niwd//ss/CrX8HixXDuuQr+Ismk/35SqebMCd09H3wQnoz1f/8XnoMrIsmnIwCpFOvWhat3jjsu3Kj197+H0TYV/EWqDyUAqVDu8Mwzobvn0UdhzJgw5PIPfqCTuyLVjbqApMIsWBC6e2bNCmPov/EG9OiR7FqJSGkSOgIws6FmtsTMlpnZr+Msf8DM5kavpWa2PmZZJzN7w8wWm9kiM+sSzX/azL6IWU+hIkVt2gTXXRcGX/vkE3jiiZAEFPxFqrdyjwDMLAN4GBgC5ACzzWyiuy8qKOPu18aUHwP0jNnEs8Cd7j7FzBoDu2OW/dLdJ+xnGyRJ3OGf/4Rf/AJyc+Gyy+D3v9fDVERSRSJHAH2AZe7+ubvvBMYDw8soPwoYB2Bm3YHa7j4FwN03u/vW/ayzVANLloS7dUeNgvbtw/NzH31UwV8klSSSADKBr2Omc6J5JZhZZ6ArMD2adRiw3sxeNLOPzOye6IiiwJ1mNj/qQqq3D/WXKrZlC9x4Y3gw+pw54bLO99+HPn2SXTMR2VsVfRXQSGCCu+dH07WBgcD1wHHAQcBF0bLfAN2i+S2BG+Jt0MwuM7NsM8tevXp1BVdXEuUOL78cLuO8667w/NwlS+CKKzQip0iqSiQBrAA6xkx3iObFM5Ko+yeSA8yNuo/ygJeBXgDunuvBDuApQldTCe4+1t2z3D2rjZ7wkRSffw5nnhmGbWjWLDxj9+mnoW3bZNdMRPZHIglgNnComXU1s7qEID+xeCEz6wa0AN4ttm5zMyuI3KcAi6Ly7aN/Dfg+sHBfGyGVY/t2uO22sNc/cybcfz98+GF4vKKIpL5yrwJy9zwzGw28DmQAT7r7x2Z2O5Dt7gXJYCQw3t09Zt18M7semBYF+jnAX6PFz0WJwYC5wOUV1irZb5Mnh5u4PvsMRo6E++6DAw9Mdq1EpCJZTLyu9rKysjw7OzvZ1ajRvvoKrr0WXnwx3M378MNhXH4RSV1mNsfds4rP11AQAsDOnXD33XDEEfDaa+FE7/z5Cv4iNZmGghBmzAhDOHzyCXz/+/Dgg9C5c7JrJSKVTUcAaSw3N1zOecop4QjgP/+Bl15S8BdJF0oAaSgvL+zlH3546Ou/5RZYuBDOOCPZNRORqqQuoDTz9tuhu2f+fDj9dHjoITjkkGTXSkSSQUcAaWL1arj44nAN/7p1Yc//P/9R8BdJZ0oANVx+fhik7fDD4bnn4Ne/Ds/jPessPaBFJN2pC6gGmz07dPdkZ4cTvX/5S7jMU0QEdARQI337bRikrW9fWLECxo2DqVMV/EWkKCWAGmT3bnjqqdDd89e/wtVXh2v7R45Ud4+IlKQEUEPMmwcDB8Ill4QE8OGH8MAD0LRpsmsmItWVEkCK27gRrrkGeveGTz8NRwAzZ8IxxyS7ZiJS3ekkcIpyh/Hjw8PYV62Cyy+HO++EFi2SXTMRSRVKAClo8WK48sowhs9xx8GkSZBVYpw/EZGyqQsohWzeDDfcELp35s4N1/e/+66Cv4jsGx0BpAD3MEjbNdfA11+HO3r/8AfQEzJFZH/oCKCaW7YsDNJ2zjmhf3/WLHjySQV/Edl/SgDV1LZtYZTOo44KA7g9+CDMmQP9+ye7ZiJSU6gLqBr6z3/gqqvg88/DeP333gvt2ye7VpLOdu3aRU5ODtu3b092VaQM9evXp0OHDtSpUyeh8koA1ciXX4a7d195JQzbMH06DBqU7FqJQE5ODk2aNKFLly6YbiuvltydtWvXkpOTQ9euXRNaR11A1cDOneEZvEccAVOmhGfzzp2r4C/Vx/bt22nVqpWCfzVmZrRq1WqvjtJ0BJBkU6fC6NGwZEk40Xv//dCpU7JrJVKSgn/1t7e/kY4AkmTFijBI25Ah4RGNkyfDhAkK/iJSdZQAqtiuXWEvv1u30Nd/223hebxDhya7ZiLV19q1a+nRowc9evSgXbt2ZGZm7pneuXNnQtu4+OKLWbJkSRsuDV0AABBeSURBVJllHn74YZ577rmKqHJKSKgLyMyGAn8CMoDH3f3uYssfAAp6rBsCbd29ebSsE/A40BFw4Ax3X25mXYHxQCtgDvAjd0/sl0xRb70VHtCycCF897vhebwHHZTsWolUf61atWLu3LkA3HrrrTRu3Jjrr7++SBl3x92pVSv+fu1TTz1V7udceeWV+1/ZFFJuAjCzDOBhYAiQA8w2s4nuvqigjLtfG1N+DNAzZhPPAne6+xQzawzsjub/AXjA3ceb2aPAT4BH9rdB1dGqVfCrX8Gzz0LnzvDyyzBsmMbol9R0zTXhIoWK1KNHuNdlby1btoxhw4bRs2dPPvroI6ZMmcJtt93Ghx9+yLZt2zj//PO5+eabARgwYAB/+ctfOOqoo2jdujWXX345kydPpmHDhrzyyiu0bduWm266idatW3PNNdcwYMAABgwYwPTp09mwYQNPPfUUJ5xwAlu2bOHCCy9k8eLFdO/eneXLl/P444/To0ePInW75ZZbePXVV9m2bRsDBgzgkUcewcxYunQpl19+OWvXriUjI4MXX3yRLl268Pvf/55x48ZRq1YtzjzzTO68886K+GrLlEgXUB9gmbt/Hu2hjweGl1F+FDAOwMy6A7XdfQqAu292960WzlScAkyI1nkG+P4+tqHays+Hhx8O4/OPGwc33giLFsHw4Qr+IhXlk08+4dprr2XRokVkZmZy9913k52dzbx585gyZQqLFi0qsc6GDRs46aSTmDdvHscffzxPPvlk3G27Ox988AH33HMPt99+OwB//vOfadeuHYsWLeK3v/0tH330Udx1r776ambPns2CBQvYsGEDr732GgCjRo3i2muvZd68ebzzzju0bduWSZMmMXnyZD744APmzZvHL37xiwr6dsqWSBdQJvB1zHQO0DdeQTPrDHQFpkezDgPWm9mL0fypwK+BFsB6d8+L2WZmKdu8DLgMoFMKnSF9//3Q3fPhhzB4cHgeb7duya6VyP7blz31ynTwwQeTFTMi4rhx43jiiSfIy8tj5cqVLFq0iO7duxdZp0GDBpx++ukA9O7dm7feeivuts8+++w9ZZYvXw7ArFmzuOGGGwA49thjOfLII+OuO23aNO655x62b9/OmjVr6N27N/369WPNmjV873vfA8KNWwBTp07lkksuoUGDBgC0bNlyX76KvVbRJ4FHAhPcPT+arg0MBK4HjgMOAi7amw26+1h3z3L3rDYpMADO2rXws5/B8cfDN9+EMfunTFHwF6ksjRo12vP+008/5U9/+hPTp09n/vz5DB06NO518XXr1t3zPiMjg7y8vBJlAOrVq1dumXi2bt3K6NGjeemll5g/fz6XXHJJtbyLOpEEsIJwArdAh2hePCOJun8iOcDcqPsoD3gZ6AWsBZqbWcERSFnbTAm7d8MTT4TunieegGuvDc/jPf98dfeIVJWNGzfSpEkTmjZtSm5uLq+//nqFf0b//v15/vnnAViwYEHcLqZt27ZRq1YtWrduzaZNm3jhhRcAaNGiBW3atGHSpElAuMFu69atDBkyhCeffJJt27YB8O2331Z4veNJJAHMBg41s65mVpcQ5CcWL2Rm3QhdO+8WW7e5mRXsup8CLHJ3B2YAI6L5PwZe2bcmJN/cuTBgAFx6KXTvDh99BPfdB02aJLtmIumlV69edO/enW7dunHhhRfSvxJGTxwzZgwrVqyge/fu3HbbbXTv3p1mzZoVKdOqVSt+/OMf0717d04//XT69i3sNX/uuee47777OOaYYxgwYACrV6/mzDPPZOjQoWRlZdGjRw8eeOCBCq93XAWXTpX1As4AlgKfAf8vmnc7MCymzK3A3XHWHQLMBxYATwN1o/kHAR8Ay4B/AfXKq0fv3r29Olm/3n3MGPdatdzbtnV/5hn33buTXSuRirdo0aJkV6Ha2LVrl2/bts3d3ZcuXepdunTxXbt2JblWheL9VkC2x4mpCd0H4O6vAq8Wm3dzselbS1l3ClDiEeXu/jnhCqOU4w7PPQfXXw+rV8MVV8Add0Dz5smumYhUts2bNzN48GDy8vJwdx577DFq107NUXVSs9ZJ9PHH4Xm8b74JffrAq69Cr17JrpWIVJXmzZszZ86cZFejQmgoiARt3gy//GW4YWXBAnjssfA8XgV/EUlVOgIohzu88EK4qicnB37ykzBcc+vWya6ZiMj+0RFAGT79NAzSdu65IeC/8w48/riCv4jUDEoAcWzdCr/9bXge73vvhUHbZs8ON3eJiNQUSgDFTJoERx4Zruo577zwoJYxYyBFT/KL1AiDBg0qcVPXgw8+yBVXXFHmeo0bNwZg5cqVjBgxIm6Zk08+mezs7DK38+CDD7J169Y902eccQbr169PpOrVmhJA5Isvwgidw4ZBw4bw3//C3/4G7dolu2YiMmrUKMaPH19k3vjx4xk1alRC6x944IFMmDCh/IKlKJ4AXn31VZrXgOu+036/dscOuOceuPNOyMgI76++GurUSXbNRKqpJIwHPWLECG666SZ27txJ3bp1Wb58OStXrmTgwIFs3ryZ4cOHs27dOnbt2sUdd9zB8OFFByxevnw5Z555JgsXLmTbtm1cfPHFzJs3j27duu0ZfgHgiiuuYPbs2Wzbto0RI0Zw22238dBDD7Fy5UoGDRpE69atmTFjBl26dCE7O5vWrVtz//337xlN9NJLL+Waa65h+fLlnH766QwYMIB33nmHzMxMXnnllT2DvRWYNGkSd9xxBzt37qRVq1Y899xzHHDAAWzevJkxY8aQnZ2NmXHLLbdwzjnn8Nprr3HjjTeSn59P69atmTZt2n597WmdAN54IzyP99NPYcQIeOAB6NAh2bUSkeJatmxJnz59mDx5MsOHD2f8+PGcd955mBn169fnpZdeomnTpqxZs4Z+/foxbNiwUp+P+8gjj9CwYUMWL17M/Pnz6RVzLfedd95Jy5Ytyc/PZ/DgwcyfP5+rrrqK+++/nxkzZtC62BUgc+bM4amnnuL999/H3enbty8nnXQSLVq04NNPP2XcuHH89a9/5bzzzuOFF17ghz/8YZH1BwwYwHvvvYeZ8fjjj/PHP/6R++67j9/97nc0a9aMBQsWALBu3TpWr17NT3/6U2bOnEnXrl0rZLygtEwAOTlw3XXwr3/BoYfC66/Daaclu1YiKSJJ40EXdAMVJIAnnngCCMPZ3HjjjcycOZNatWqxYsUKVq1aRbtS+m9nzpzJVVddBcAxxxzDMccUDlTw/PPPM3bsWPLy8sjNzWXRokVFlhc3a9YszjrrrD0jkp599tm89dZbDBs2jK5du+55SEzscNKxcnJyOP/888nNzWXnzp107doVCMNDx3Z5tWjRgkmTJnHiiSfuKVMRQ0an1TmAXbvg3nvD0MyTJsHvfhdu6lLwF6n+hg8fzrRp0/jwww/ZunUrvXv3BsLgaqtXr2bOnDnMnTuXAw44YJ+GXv7iiy+49957mTZtGvPnz+e73/3ufg3hXDCUNJQ+nPSYMWMYPXo0CxYs4LHHHqvyIaPTJgG8+Sb07Bnu5j3llPBkrptugpjfSESqscaNGzNo0CAuueSSIid/N2zYQNu2balTpw4zZszgyy+/LHM7J554Iv/4xz8AWLhwIfPnzwfCUNKNGjWiWbNmrFq1ismTJ+9Zp0mTJmzatKnEtgYOHMjLL7/M1q1b2bJlCy+99BIDBw5MuE0bNmwgMzM8C+uZZ57ZM3/IkCE8/PDDe6bXrVtHv379mDlzJl988QVQMUNGp0UC+NnP4OSTYcsWmDgxvKKjKBFJIaNGjWLevHlFEsAPfvADsrOzOfroo3n22WfpVs7Tl6644go2b97MEUccwc0337znSOLYY4+lZ8+edOvWjQsuuKDIUNKXXXYZQ4cOZdCgQUW21atXLy666CL69OlD3759ufTSS+nZsyeJuvXWWzn33HPp3bt3kfMLN910E+vWreOoo47i2GOPZcaMGbRp04axY8dy9tlnc+yxx3L++ecn/DmlsTBSaGrIysry8q7XjeePf4RNm+A3vwmXeIrI3lm8eDFHHHFEsqshCYj3W5nZHHfPKl42LU4C/+pXya6BiEj1kxZdQCIiUpISgIgkJJW6i9PV3v5GSgAiUq769euzdu1aJYFqzN1Zu3Yt9evXT3idtDgHICL7p0OHDuTk5LB69epkV0XKUL9+fTrsxXAGSgAiUq46dersuQNVag51AYmIpCklABGRNKUEICKSplLqTmAzWw2UPdBH6VoDayqwOslUU9pSU9oBakt1VVPasr/t6OzubYrPTKkEsD/MLDverdCpqKa0paa0A9SW6qqmtKWy2qEuIBGRNKUEICKSptIpAYxNdgUqUE1pS01pB6gt1VVNaUultCNtzgGIiEhR6XQEICIiMZQARETSVI1LAGY21MyWmNkyM/t1nOX1zOyf0fL3zaxL1deyfAm04yIzW21mc6PXpcmoZyLM7Ekz+5+ZLSxluZnZQ1Fb55tZr6quYyISaMfJZrYh5je5uarrmCgz62hmM8xskZl9bGZXxylT7X+XBNuREr+LmdU3sw/MbF7UltvilKnY+OXuNeYFZACfAQcBdYF5QPdiZX4OPBq9Hwn8M9n13sd2XAT8Jdl1TbA9JwK9gIWlLD8DmAwY0A94P9l13sd2nAz8O9n1TLAt7YFe0fsmwNI4f2PV/ndJsB0p8btE33Pj6H0d4H2gX7EyFRq/atoRQB9gmbt/7u47gfHA8GJlhgPPRO8nAIPNzKqwjolIpB0pw91nAt+WUWQ48KwH7wHNzax91dQucQm0I2W4e667fxi93wQsBjKLFav2v0uC7UgJ0fe8OZqsE72KX6VTofGrpiWATODrmOkcSv4x7Cnj7nnABqBVldQucYm0A+Cc6NB8gpl1rJqqVYpE25sKjo8O4Seb2ZHJrkwiom6EnoQ9zlgp9buU0Q5Ikd/FzDLMbC7wP2CKu5f6m1RE/KppCSCdTAK6uPsxwBQK9wokeT4kjLlyLPBn4OUk16dcZtYYeAG4xt03Jrs++6qcdqTM7+Lu+e7eA+gA9DGzoyrz82paAlgBxO4Jd4jmxS1jZrWBZsDaKqld4spth7uvdfcd0eTjQO8qqltlSOR3q/bcfWPBIby7vwrUMbPWSa5WqcysDiFoPufuL8YpkhK/S3ntSLXfBcDd1wMzgKHFFlVo/KppCWA2cKiZdTWzuoSTJBOLlZkI/Dh6PwKY7tEZlWqk3HYU64sdRuj7TFUTgQujq076ARvcPTfZldpbZtauoD/WzPoQ/n9Vt50LIFzhAzwBLHb3+0spVu1/l0TakSq/i5m1MbPm0fsGwBDgk2LFKjR+1ahHQrp7npmNBl4nXEnzpLt/bGa3A9nuPpHwx/I3M1tGOKE3Mnk1ji/BdlxlZsOAPEI7LkpahcthZuMIV2K0NrMc4BbCCS7c/VHgVcIVJ8uArcDFyalp2RJoxwjgCjPLA7YBI6vhzkWB/sCPgAVRnzPAjUAnSKnfJZF2pMrv0h54xswyCEnqeXf/d2XGLw0FISKSpmpaF5CIiCRICUBEJE0pAYiIpCklABGRNKUEICKSppQARETSlBKAiEia+v/8GERYdHScoQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxU1Zn/8c9DszRLs4PIooAbNHvTAgaRxWUQo4xKDKgxOi7RxKhJnAlxjBoSM2oMGg3jyPiTaFyIo1ExgMQIiEsEGkQQAUFB2WRTdhAant8fp7qraKq7q+mluqq/79erXlTdOnXruV36nHPPufccc3dERCR91Up2ACIiUrmU6EVE0pwSvYhImlOiFxFJc0r0IiJpToleRCTNKdFLmZhZhpntNrMTKrJsMpnZyWZW4dcZm9k5ZrYm5vUKMxuUSNlj+K4nzOyOY/18Cfv9jZn9qaL3K1WrdrIDkMplZrtjXjYAvgEORV7/wN2fLcv+3P0Q0Kiiy9YE7n5aRezHzK4DrnT3ITH7vq4i9i3pSYk+zbl7YaKNtBivc/d/FFfezGq7e35VxCYiVUNdNzVc5NT8L2b2vJntAq40szPM7H0z225mG83sETOrEylf28zczDpGXj8TeX+6me0ys3+aWaeylo28f76ZfWJmO8zsUTN718yuLibuRGL8gZmtMrOvzeyRmM9mmNlDZrbNzD4Dhpfw9/lPM5tcZNsEMxsfeX6dmS2LHM+nkdZ2cftaZ2ZDIs8bmNmfI7EtBfoWKXunmX0W2e9SM7sosr0H8EdgUKRbbGvM3/aemM/fGDn2bWb2ipkdn8jfpjRmdnEknu1mNtPMTot57w4z22BmO81secyxDjCzhZHtm8zsd4l+n1QQd9ejhjyANcA5Rbb9BjgAXEio+OsDpwP9CWd8nYFPgJsj5WsDDnSMvH4G2ArkAnWAvwDPHEPZ1sAuYGTkvZ8CB4GrizmWRGJ8FWgCdAS+Kjh24GZgKdAeaAHMCf8rxP2ezsBuoGHMvjcDuZHXF0bKGDAM2Af0jLx3DrAmZl/rgCGR5w8Cs4FmwInAx0XKXgYcH/lNLo/EcFzkveuA2UXifAa4J/L8vEiMvYFM4L+BmYn8beIc/2+AP0Wed43EMSzyG90BrIg87wZ8DrSJlO0EdI48nw+MiTzPAvon+/+FmvZQi14A3nH319z9sLvvc/f57j7X3fPd/TNgIjC4hM+/6O557n4QeJaQYMpa9tvAInd/NfLeQ4RKIa4EY/wvd9/h7msISbXguy4DHnL3de6+DbivhO/5DPiIUAEBnAt87e55kfdfc/fPPJgJvAnEHXAt4jLgN+7+tbt/Tmilx37vC+6+MfKbPEeopHMT2C/AFcAT7r7I3fcDY4HBZtY+pkxxf5uSjAamuPvMyG90H6Gy6A/kEyqVbpHuv9WRvx2ECvsUM2vh7rvcfW6CxyEVRIleANbGvjCzLmY21cy+NLOdwDigZQmf/zLm+V5KHoAtrmzb2Djc3Qkt4LgSjDGh7yK0REvyHDAm8vzyyOuCOL5tZnPN7Csz205oTZf0typwfEkxmNnVZvZhpItkO9Alwf1COL7C/bn7TuBroF1MmbL8ZsXt9zDhN2rn7iuAnxF+h82RrsA2kaLXANnACjObZ2YjEjwOqSBK9ALhVD7W44RW7Mnu3hi4i9A1UZk2ErpSADAz48jEVFR5YtwIdIh5Xdrlny8A55hZO0LL/rlIjPWBF4H/InSrNAX+nmAcXxYXg5l1Bh4DbgJaRPa7PGa/pV0KuoHQHVSwvyxCF9H6BOIqy35rEX6z9QDu/oy7DyR022QQ/i64+wp3H03onvs98JKZZZYzFikDJXqJJwvYAewxs67AD6rgO/8G5JjZhWZWG7gVaFVJMb4A3GZm7cysBfDzkgq7+5fAO8CfgBXuvjLyVj2gLrAFOGRm3wbOLkMMd5hZUwv3Gdwc814jQjLfQqjzrie06AtsAtoXDD7H8TxwrZn1NLN6hIT7trsXe4ZUhpgvMrMhke/+d8K4ylwz62pmQyPfty/yOEw4gO+ZWcvIGcCOyLEdLmcsUgZK9BLPz4DvE/4nfpwwaFqp3H0T8F1gPLANOAn4gHDdf0XH+BihL30JYaDwxQQ+8xxhcLWw28bdtwM/AV4mDGiOIlRYibibcGaxBpgOPB2z38XAo8C8SJnTgNh+7TeAlcAmM4vtgin4/OuELpSXI58/gdBvXy7uvpTwN3+MUAkNBy6K9NfXAx4gjKt8STiD+M/IR0cAyyxc1fUg8F13P1DeeCRxFrpCRaoXM8sgdBWMcve3kx2PSCpTi16qDTMbHunKqAf8knC1xrwkhyWS8pTopTo5E/iM0C3wL8DF7l5c142IJEhdNyIiaU4tehGRNFftJjVr2bKld+zYMdlhiIiklAULFmx197iXJFe7RN+xY0fy8vKSHYaISEoxs2Lv8FbXjYhImlOiFxFJc0r0IiJprtr10YtI1Tp48CDr1q1j//79yQ5FEpCZmUn79u2pU6e4qY6OpkQvUsOtW7eOrKwsOnbsSJg0VKord2fbtm2sW7eOTp06lf6BCHXdiNRw+/fvp0WLFkryKcDMaNGiRZnPvpToRURJPoUcy2+VNol+/3649VbYsCHZkYiIVC9pk+jnz4eJE6FLF/jjH+HQoWRHJCKJ2LZtG71796Z37960adOGdu3aFb4+cCCxaeuvueYaVqxYUWKZCRMm8Oyzz1ZEyJx55pksWrSoQvZVFdJmMHbQIFiyBH74Q/jxj+Hpp0Pi753IkscikjQtWrQoTJr33HMPjRo14vbbbz+ijLvj7tSqFb9tOmnSpFK/50c/+lH5g01RadOiBzj5ZJgxA557Dj7/HHJz4fbbYffuZEcmImW1atUqsrOzueKKK+jWrRsbN27khhtuIDc3l27dujFu3LjCsgUt7Pz8fJo2bcrYsWPp1asXZ5xxBps3bwbgzjvv5OGHHy4sP3bsWPr168dpp53Ge++9B8CePXu49NJLyc7OZtSoUeTm5pbacn/mmWfo0aMH3bt354477gAgPz+f733ve4XbH3nkEQAeeughsrOz6dmzJ1deeWWF/82KkzYt+gJmMGYMDB8OY8fC738PL7wAEybAhRcmOzqR6u2226CieyR694ZIfi2z5cuX8/TTT5ObmwvAfffdR/PmzcnPz2fo0KGMGjWK7OzsIz6zY8cOBg8ezH333cdPf/pTnnzyScaOHXvUvt2defPmMWXKFMaNG8frr7/Oo48+Sps2bXjppZf48MMPycnJKTG+devWceedd5KXl0eTJk0455xz+Nvf/karVq3YunUrS5YsAWD79u0APPDAA3z++efUrVu3cFtVSKsWfaxmzeDxx+Gdd6BxY7joIrjkElhX3uWRRaTKnHTSSYVJHuD5558nJyeHnJwcli1bxscff3zUZ+rXr8/5558PQN++fVmzZk3cfV9yySVHlXnnnXcYPXo0AL169aJbt24lxjd37lyGDRtGy5YtqVOnDpdffjlz5szh5JNPZsWKFdxyyy3MmDGDJk2aANCtWzeuvPJKnn322TLd8FReadeiL2rgQFi4EMaPh3HjoGtXuPde+NGPICMj2dGJVC/H2vKuLA0bNix8vnLlSv7whz8wb948mjZtypVXXhn3evK6desWPs/IyCA/Pz/uvuvVq1dqmWPVokULFi9ezPTp05kwYQIvvfQSEydOZMaMGbz11ltMmTKF3/72tyxevJiMKkhEaduij1W3bujG+eijkPhvvRUGDAgVgIikhp07d5KVlUXjxo3ZuHEjM2bMqPDvGDhwIC+88AIAS5YsiXvGEKt///7MmjWLbdu2kZ+fz+TJkxk8eDBbtmzB3fnOd77DuHHjWLhwIYcOHWLdunUMGzaMBx54gK1bt7J3794KP4Z40r5FH6tzZ5g+PfTZ33ornH463HJLaOlnZSU7OhEpSU5ODtnZ2XTp0oUTTzyRgQMHVvh3/PjHP+aqq64iOzu78FHQ7RJP+/bt+fWvf82QIUNwdy688EIuuOACFi5cyLXXXou7Y2bcf//95Ofnc/nll7Nr1y4OHz7M7bffTlYVJZ5qt2Zsbm6uV8XCI9u3wy9+Efrx27UL196PHFnpXytS7SxbtoyuXbsmO4xqIT8/n/z8fDIzM1m5ciXnnXceK1eupHbt6tUmjvebmdkCd8+NV75GdN3E07QpPPYYvPtuGLj9138Nj7Vrkx2ZiCTL7t27GThwIL169eLSSy/l8ccfr3ZJ/lik/hGU0xlnwIIF8NBDcM89YbD2N7+Bm2+GNPh9RaQMmjZtyoIFC5IdRoWrsS36WHXqwH/8ByxdCoMHw09+Av37g5auFZF0oEQfo1Mn+Nvf4P/+DzZuDMn+1lth585kRyYicuyU6Iswg1GjYNkyuOkmePTR0J3z179CNRu3FhFJiBJ9MZo0CVfi/POf0KoVXHppuCrn88+THZmISNko0ZeioK/+wQfhzTchOzvMn1PBN9KJ1FhDhw496uanhx9+mJtuuqnEzzVq1AiADRs2MGrUqLhlhgwZQmmXaz/88MNH3Lg0YsSICpmH5p577uHBBx8s934qQkKJ3syGm9kKM1tlZkfPDhQtd6mZuZnlRl7XNbNJZrbEzD40syEVFHeVql0bfvYz+PhjGDo0zIh5+ukwb16yIxNJfWPGjGHy5MlHbJs8eTJjxoxJ6PNt27blxRdfPObvL5rop02bRtOmTY95f9VRqYnezDKACcD5QDYwxsyy45TLAm4F5sZsvh7A3XsA5wK/N7OUPYs48UR47TV48UXYvDlMo3DzzbBjR7IjE0ldo0aNYurUqYWLjKxZs4YNGzYwaNAgdu/ezdlnn01OTg49evTg1VdfPerza9asoXv37gDs27eP0aNH07VrVy6++GL27dtXWO6mm24qnOL47rvvBuCRRx5hw4YNDB06lKFDhwLQsWNHtm7dCsD48ePp3r073bt3L5zieM2aNXTt2pXrr7+ebt26cd555x3xPfEsWrSIAQMG0LNnTy6++GK+/vrrwu8vmLa4YDK1t956q3DhlT59+rBr165j/tsWSORK8X7AKnf/DMDMJgMjgaKTQPwauB/495ht2cBMAHffbGbbgVwgZdvCZqG//txz4c47Qz/+X/8KjzwStmvpTUlpSZinuHnz5vTr14/p06czcuRIJk+ezGWXXYaZkZmZycsvv0zjxo3ZunUrAwYM4KKLLip23dTHHnuMBg0asGzZMhYvXnzENMP33nsvzZs359ChQ5x99tksXryYW265hfHjxzNr1ixatmx5xL4WLFjApEmTmDt3Lu5O//79GTx4MM2aNWPlypU8//zz/O///i+XXXYZL730Uonzy1911VU8+uijDB48mLvuuotf/epXPPzww9x3332sXr2aevXqFXYXPfjgg0yYMIGBAweye/duMjMzy/LXjiuR1nU7IPZ+0XWRbYXMLAfo4O5Ti3z2Q+AiM6ttZp2AvkCHol9gZjeYWZ6Z5W3ZsqVMB5AsjRuH5D53LrRpA9/5TpjvvpgZUUWkBLHdN7HdNu7OHXfcQc+ePTnnnHNYv349mzZtKnY/c+bMKUy4PXv2pGfPnoXvvfDCC+Tk5NCnTx+WLl1a6oRl77zzDhdffDENGzakUaNGXHLJJbz99tsAdOrUid6R5etKmgoZwvz427dvZ/DgwQB8//vfZ86cOYUxXnHFFTzzzDOFd+AOHDiQn/70pzzyyCNs3769Qu7MLfceIl0x44Gr47z9JNAVyAM+B94DjlrN1d0nAhMhzHVT3piqUkFf/aOPwi9/Cd26hTtsb7st3IglklKSNE/xyJEj+clPfsLChQvZu3cvffv2BeDZZ59ly5YtLFiwgDp16tCxY8e4UxOXZvXq1Tz44IPMnz+fZs2acfXVVx/TfgoUTHEMYZrj0rpuijN16lTmzJnDa6+9xr333suSJUsYO3YsF1xwAdOmTWPgwIHMmDGDLl26HHOskFiLfj1HtsLbR7YVyAK6A7PNbA0wAJhiZrnunu/uP3H33u4+EmgKfFKuiKuh2rXD3bQffwznnBPuss3NhfffT3ZkIqmhUaNGDB06lH/7t387YhB2x44dtG7dmjp16jBr1iw+L+X65rPOOovnnnsOgI8++ojFixcDYYrjhg0b0qRJEzZt2sT06dMLP5OVlRW3H3zQoEG88sor7N27lz179vDyyy8zaNCgMh9bkyZNaNasWeHZwJ///GcGDx7M4cOHWbt2LUOHDuX+++9nx44d7N69m08//ZQePXrw85//nNNPP53ly5eX+TuLSqRFPx84JdL1sh4YDVxe8Ka77wAKO7fMbDZwu7vnmVkDwgyZe8zsXCDf3Us+X0phJ5wAr74Kr7wSBmm/9S248Ub47W/DJGoiUrwxY8Zw8cUXH3EFzhVXXMGFF15Ijx49yM3NLbVle9NNN3HNNdfQtWtXunbtWnhm0KtXL/r06UOXLl3o0KHDEVMc33DDDQwfPpy2bdsya9aswu05OTlcffXV9OvXD4DrrruOPn36lNhNU5ynnnqKG2+8kb1799K5c2cmTZrEoUOHuPLKK9mxYwfuzi233ELTpk355S9/yaxZs6hVqxbdunUrXC2rPBKaptjMRgAPAxnAk+5+r5mNA/LcfUqRsrOJJvqOwAzgMKGSuNbdS6ySq2qa4sq2axfcdVfox2/dOpwRX3aZBmul+tE0xamnUqYpdvdp7n6qu5/k7vdGtt1VNMlHtg9x97zI8zXufpq7d3X3c0pL8ukkKyvMiDlvHrRtC6NHw4gRsHp1siMTkZomZa9pTxV9+4Yrcx5+OCxU3q0b3H8/HDyY7MhEpKZQoq8CtWuHWTCXLYPhw8P6tTk58N57yY5MJKhuK81J8Y7lt1Kir0Lt24ebq159NdxNO3BgGKyN3CQnkhSZmZls27ZNyT4FuDvbtm0r801UNXbN2GTbvRvuvjt06bRqFfrzR4/WYK1UvYMHD7Ju3bpyXVcuVSczM5P27dtTp8iNOiUNxirRJ9kHH8APfgDz58N558F//zecdFKyoxKRVKPFwauxPn3CnPePPhr+7d49XHcfmd9JRKTclOirgYyMcIPVsmVwwQXwn/8ZKoB33kl2ZCKSDpToq5F27cIUyK+9FvrwBw2C66+Hr75KdmQiksqU6Kuhb387zJtz++0waRJ06QLPPqs1a0Xk2CjRV1MNG8LvfgcLFkDnznDllWGwdtWqZEcmIqlGib6a69UL3n0XJkwI0yl07w6/+Q18802yIxORVKFEnwIyMuCHPwyDtSNHhnnv+/SByNoFIiIlUqJPIW3bwl/+AlOnwr59MHgwXHstbNuW7MhEpDpTok9BI0bA0qXw85/DU0+Fwdqnn9ZgrYjEp0Sfoho0gPvug4UL4eST4fvfD6tbfZJ263eJSHkp0ae4nj3DYO1jj4UrdHr0gF/9SoO1IhKlRJ8GatUKs2AuXw6XXBIWJ+/VC2bPTnZkIlIdKNGnkTZt4PnnYfr0MFfO0KFwzTWwdWuyIxORZFKiT0PDh8NHH8EvfgHPPBMGa//0Jw3WitRUSvRpqkGDMAvmBx/AaaeFlv3QoaF7R0RqFiX6NNe9O7z9NkycCB9+GPru774btMaESM2hRF8D1KoVZsFcvhxGjYJx48LVOjNnJjsyEakKSvQ1yHHHhVkw//53OHwYzj4brroKtmxJdmQiUpmU6Gugc8+FJUvCAieTJ4fB2ief1GCtSLpSoq+h6tcPs2AuWgTZ2WHOnCFDwsRpIpJelOhruOxseOsteOKJ0Mrv1SvMjrlvX7IjE5GKokQv1KoVWvTLl8N3vxta+j17wj/+kezIRKQiKNFLodat4c9/hjfeCK/PPTesbLV5c3LjEpHyUaKXo5xzTujGuesueOGFMFj7xBPhSh0RST1K9BJXZmaYBfPDD8OMmNdfD2edFebBF5HUokQvJeraNcyC+eST4Yqc3r3DZZkarBVJHUr0UiqzMFfO8uVwxRVhDp3u3WHGjGRHJiKJUKKXhLVqFWbBnDkTatcOs2Refjl8+WWyIxORkijRS5kNHRr67u+5B156KXTvPP64BmtFqquEEr2ZDTezFWa2yszGllDuUjNzM8uNvK5jZk+Z2RIzW2Zmv6iowCW5MjPDLJiLF4d++xtvhDPPDFfriEj1UmqiN7MMYAJwPpANjDGz7DjlsoBbgbkxm78D1HP3HkBf4Adm1rH8YUt1cdppoSvnqafCwuQ5OTB2LOzdm+zIRKRAIi36fsAqd//M3Q8Ak4GRccr9GrgfiJ3p3IGGZlYbqA8cAHaWL2SpbszCLJjLl8P3vgf33w/duoUlDUUk+RJJ9O2AtTGv10W2FTKzHKCDu08t8tkXgT3ARuAL4EF3/+rYw5XqrGXLcBnm7Nmha2fEiDClwurVyY5MpGYr92CsmdUCxgM/i/N2P+AQ0BboBPzMzDrH2ccNZpZnZnlbNDl6yhs8OMyKOW4cvPoqdO4Mp5wCP/gB/OUvsGlTsiMUqVkSSfTrgQ4xr9tHthXIAroDs81sDTAAmBIZkL0ceN3dD7r7ZuBdILfoF7j7RHfPdffcVq1aHduRSLVSr16YBXPZMhg/PlyZM3kyjB4NbdqE6/B//GN4+WX4Sud4IpXKvJTVJiL9658AZxMS/HzgcnePezO8mc0Gbnf3PDP7OdDF3a8xs4aRz45298XFfV9ubq7n5eUd08FI9ZafDwsXwqxZYQD37bfDHbZm4cqdYcPCpZuDBkHjxsmOViS1mNkCdz+qIQ0JJPrIDkYADwMZwJPufq+ZjQPy3H1KkbKziSb6RsAkwtU6Bkxy99+V9F1K9DXHgQMwb15I+rNmwXvvhW0ZGXD66SHpDxsG3/oWNGiQ7GhFqrdyJ/qqpERfc+3bF5J9QYt/3jw4dAjq1oUBA6It/v79Q9eQiEQp0UtK2rUL3nkn2uJfuDCsa1u/frg5q6DF37dvmJJBpCZTope08PXXMGdOSPwzZ8JHH4XtWVlhCuWCFn+vXmHVLJGapKREr3aQpIxmzWDkyPCAsPLV7NnRrp6pkbs4mjcPl3gOGxYeXbuGAV+Rmkotekkb69dHk/7MmfD552H7cceFln5BV89JJynxS/pR143USKtXR/v3Z86EjRvD9g4dokl/6FA44YTkxilSEZTopcZzhxUrokl/9mzYujW8d9JJ0aQ/dGi4oUsk1SjRixRx+HAYzC1o8c+eDTsj0+117Rrt3x88GFq0SGqoIglRohcpxaFD8MEH0f79t98OUy2bhat4Clr8Z52lu3alelKiFymjAwdg/vxoV89778E334S7dvv2jbb4Bw7UXbtSPSjRi5TT/v3wz39GW/zz5oW5e+rUCXftFgzuDhigu3YlOZToRSrY7t3hrt2CFv/ChaHfPzMztPILunpyc0NlIFLZlOhFKtn27dG7dmfNCmvpAjRqFPr1C1r8vXqF7h+RiqY7Y0UqWdOmcNFF4QGwZcuRd+1OmxYtN2RItMXfrZtu3pLKpxa9SBXYsCGa9GfNii6v2Lp1NPEPGwYnn6zEL8dGXTci1cyaNUdO17BhQ9jerl20tT9sGJx4YlLDlBSiRC9SjbnDypXR1v6sWaHrB8J6u7HTNRx/fHJjlepLiV4khRw+DEuXRlv8b70VBnsBunSJJv0hQ6Bly6SGKtWIEr1ICjt0CBYtirb458yBPXvCez17Rvv3zzoLmjRJbqySPEr0Imnk4MGj79rdvz8sttK3b7Sr58wzoWHDZEcrVUWJXiSN7d8P778fbfG//370rt1+/aJdPWecEW7okvSkRC9Sg+zZA+++G72iZ8GC0O9fr164a3fo0DArZ3a2ZuZMJ0r0IjXYjh2hX7+gq+fDD6PvNWsGp5wS/9G0afJilrJToheRQlu3hgnaPvkkXNZZ8Fi79shyLVsWXwlkZSUndimepkAQkUItW8KFFx69fd8++PTTaOJftSr8++ab8PTTR5Y97rj4FcDJJ2sAuDpSohcRAOrXh+7dw6OoPXuOrAQKHtOnw6RJR5Zt2zZ+JXDSSeE7pOop0YtIqRo2DNfs9+x59Hu7dkVb/7GPV1+N3uFboEOHo88ACioBzeNfeZToRaRcsrKgT5/wKGrHjqMrgJUr4f/+D776KlrODE44If6ZQKdOULdu1R1POlKiF5FK06RJWHwlN84Q4Vdfxa8EnnsuVBAFMjLC5G7xKoGOHaG2slip9CcSkaRo3hz69w+PWO6wbVv8SuC990JXUYHatUOLP14lcMIJWuSlgBK9iFQrZuHKoJYtw928sdxh8+b4lcDs2bB3b7Rs3bph9s94lUD79mHKiJpCiV5EUoZZuLTzuOPCXD6x3GHjxviVwBtvhKkiCmRmhgHgeAPD7dql3+IvSvQikhbMwqWdbduGKR5iHT4M69cfXQGsWBGWeTxwIFq2fv1o0i/6aNMmNSsBJXoRSXu1aoVLOzt0CJO8xTp0KNwVXPRGsaVL4bXXwmyhBRo1Kr4SaNWq+lYCSvQiUqNlZISrdzp2hHPPPfK9/Hz44oujzwQ++AD++tdQSRRo3Lj4KSOSPXmc5roRETkGBw+GtX/jjQl8/nnoLipQFZPHlXuuGzMbDvwByACecPf7iil3KfAicLq755nZFcC/xxTpCeS4+6KyHICISHVTp040WRf1zTewevXRFcDbb4f7BGLb17GTx519Nlx1VcXHWmqiN7MMYAJwLrAOmG9mU9z94yLlsoBbgbkF29z9WeDZyPs9gFeU5EUk3dWrF9b37dLl6Pf2748/b9Cbb4ZupKQkeqAfsMrdPwMws8nASODjIuV+DdzPkS34WGOAyccYp4hIWsjMhG7dwqOo2O6eipTILQPtgNiZqtdFthUysxygg7tPLWE/3wWej/eGmd1gZnlmlrel6CxIIiI1RGXdxFXu3ZpZLWA88LMSyvQH9rr7R/Hed/eJ7p7r7rmtWrUqb0giIhIjkUS/HugQ87p9ZFuBLKA7MNvM1gADgClmFjv6O5piWvMiIlK5Eumjnw+cYmadCAl+NHB5wZvuvgNoWfDazGYDt7t7XuR1LeAyYFDFhS0iIokqtUXv7vnAzcAMYBnwgrsvNbNxZnZRAt9xFrC2YDBXRESqlm6YEsVtwD8AAA77SURBVBFJAyXdMFWDJuoUEamZlOhFRNKcEr2ISJpTohcRSXNK9CIiaU6JXkQkzSnRi4ikOSV6EZE0p0QvIpLmlOhFRNKcEr2ISJpTohcRSXNK9CIiaU6JXkQkzSnRi4ikOSV6EZE0p0QvIpLmlOhFRNKcEr2ISJpTohcRSXNK9CIiaU6JXkQkzSnRi4ikOSV6EZE0p0QvIpLmlOhFRNKcEr2ISJpTohcRSXNK9CIiaU6JXkQkzSnRi4ikOSV6EZE0p0QvIpLmlOhFRNKcEr2ISJpLKNGb2XAzW2Fmq8xsbAnlLjUzN7PcmG09zeyfZrbUzJaYWWZFBH6UgwdhyhTYvbtSdi8ikqpKTfRmlgFMAM4HsoExZpYdp1wWcCswN2ZbbeAZ4EZ37wYMAQ5WSORFvfcejBwJLVrAeefBH/4AK1dWyleJiKSSRFr0/YBV7v6Zux8AJgMj45T7NXA/sD9m23nAYnf/EMDdt7n7oXLGHN8ZZ8DMmXDLLbBuHdx2G5x6anjcdhu88QZ8802lfLWISHWWSKJvB6yNeb0usq2QmeUAHdx9apHPngq4mc0ws4Vm9h/xvsDMbjCzPDPL27JlSxnCj1G3LgwdCr/7HXz8MXz2Gfzxj3DyyfD446GV36IF/Ou/wsSJoTIQEakBapd3B2ZWCxgPXF3M/s8ETgf2Am+a2QJ3fzO2kLtPBCYC5ObmenljAqBTJ/jRj8Jj716YNQumTYOpU+HVV0OZnj3hggtgxAgYMABql/vPISJS7STSol8PdIh53T6yrUAW0B2YbWZrgAHAlMiA7Dpgjrtvdfe9wDQgpyICL5MGDUJCnzABVq+GpUvhgQegefNwBjBoELRuDWPGwDPPwNatVR6iiEhlMfeSG9CRAdVPgLMJCX4+cLm7Ly2m/GzgdnfPM7NmwJuEVv0B4HXgoThdPIVyc3M9Ly/vGA7lGO3YEfrvp06F6dNh0yYwg/79Q0v/ggugd2+opStRRaT6ivSW5MZ7r9Ts5e75wM3ADGAZ8IK7LzWzcWZ2USmf/ZrQrTMfWAQsLCnJJ0WTJjBqFEyaBBs2wPz5cM89cPgw3H039O0L7drBtdfCSy/Bzp3JjlhEpExKbdFXtSpv0Zdk82Z4/fXQtz9jBmzfHvrxBw2K9u136RLOAEREkqikFr0SfaLy8+Gf/wxdPNOmwZIlYXunTtEuniFDoH79pIYpIjWTEn1l+OKL0Kc/dSq8+Wa4sqd+fRg2LJr4Tzwx2VGKSA2hRF/Z9u+Ht96KXr756adhe3Z2tItn4ECoUye5cYpI2lKir0ruYeqFgi6et94K8/A0bhxu2rrgAhg+HNq0SXakIpJGlOiTadeu0LVTkPg3bAjbc3OjXTy5ubp8U0TKRYm+unCHDz+MdvG8/364jLNVKzj//JD4/+VfoGnTZEcqIilGib662rYtXLY5bVoY2P3qK8jIgG99K9q33727Lt8UkVIp0aeCQ4dg3rxoF88HH4TtHTpEu3iGDYOGDZMbp4hUS0r0qWjDhujlm2+8ERZUqVcvXKtfkPhPOinZUYpINaFEn+oOHIC334727a9YEbafemq0i+ess8JUzSJSIynRp5tPP40m/dmzw4IqjRrBOeeExH/++WF+HhGpMZTo09mePWFlrYLEvzayRkzv3tEunv79wyCviKQtJfqawj3MtV8woPvuu2GQt3nzcJPWiBHh3xYtkh2piFQwJfqa6uuvj5xrf8uWcGNW//7Rvv3evXX5pkgaUKKXcGNWXl60i6fgb9y2bUj4I0aEPv6srOTGKSLHRIlejrZpU2jlF8y1v3NnmHTtrLOiffunnqrWvkiKUKKXkh08CO+9F+3bXxpZJbJz52gXz5AhkJmZ1DBFpHhK9FI2a9ZEb9aaORP27QsLrJ99drSb54QTkh2liMRQopdjt29fuFa/oG9/9eqwvXv3aBfPGWdorn2RJFOil4rhHu7KLejimTMnLLHYpEmYdXPEiHCzVuvWyY5UpMZRopfKsXMn/OMf0cT/5Zdh8Pb006NdPH37aq59kSqgRC+V7/BhWLQo2sUzd244A2jdGvr1C3Puxz5atz7ydYMGyT4CkZSmRC9Vb+tWeP31kPiXLQs3a23ZEiZoi6dBg/gVQHEVQ8OGuvRTJIYSvVQP7mFpxc2bo4k/9hFv+/798feVmVm2iiErSxWDpLWSEn3tqg5GajCzsEh648Zw8smll3cPk7YVVwnEbi84a9i7N/6+6tYtW8XQpIkqBkkbSvRSfZmF6ZcbNYJOnRL7zN69iVUMK1eGf3fvjr+fOnWgZcvEK4amTTXoLNWWEr2klwYN4MQTwyMR+/aF8YTSKobVq8O/O3fG309GRtkqhubNVTFIlVGil5qtfv2wLm+HDomV/+abxCqGhQvDv9u3x99PrVphuuhEK4YWLbSmgBwzJXqRsqhXL6zelegKXgcPJlYxLF4c/v3qq/j7MQtnAYlWDC1bQm397y2B/ksQqUx16sDxx4dHIvLzYdu20iuGjz8O/27bFgat42nWrOSKoX37cCbTrp2msEhzSvQi1Unt2nDcceGRiEOHwllAaRXDJ5+EFce2bg03t8UygzZtwkR1Bd1YRR/HHaeuoxSmRC+SyjIyoq30RBw+HFYe27QprC9c9LFkSbjJrehlqrVrh5Z/cRVBhw6hu0iXpFZLSvQiNUnBIHCLFpCdHb+Me6gM4lUEa9eG6S1eeunou5wzM6PdQR06xD9DaNKk8o9RjqJELyJHKhj4bd4cevWKX+bw4dAl9MUX8SuDmTNhw4aju4mysko+K+jQQfMeVQIlehEpu1q1omMJp58ev0x+PmzcWPyZwaJFoQupqObNS64I2rcPdzpLwhJK9GY2HPgDkAE84e73FVPuUuBF4HR3zzOzjsAyYEWkyPvufmN5gxaRFFC7dun3KHzzDaxfH//M4IsvwgDy118f+RmzUMGUVBkcf7wGj2OUmujNLAOYAJwLrAPmm9kUd/+4SLks4FZgbpFdfOruvSsoXhFJJ/XqhbWJO3cuvsyePcWfFSxbBn//+9FTWWRkQNu2JVcGrVvXmMHjRFr0/YBV7v4ZgJlNBkYCHxcp92vgfuDfKzRCEanZGjaELl3CIx532LEjehZQtDLIy4NXXglnD7Hq1g3dQCVdVtq0aVpUBokk+nbA2pjX64D+sQXMLAfo4O5Tzaxoou9kZh8AO4E73f3tol9gZjcANwCcoEWnRaQszEJCbtoUevSIX8Y9DB4Xd2bw1luhC+nQoSM/17Bh6YPHjRpV/jGWU7kHY82sFjAeuDrO2xuBE9x9m5n1BV4xs27ufsTMUO4+EZgIYT768sYkInIEs9BV07p1WN4ynkOHwnKYsWMERe8x2LTp6DuRmzaNXwEUnCm0bx+6qJIokUS/HogdTWkf2VYgC+gOzLZwitMGmGJmF7l7HvANgLsvMLNPgVMBrSwiItVLRkZ0HqMBA+KXOXAgtPxLusdg27ajP9e6dclnBW3bVurcRInseT5wipl1IiT40cDlBW+6+w6gZcFrM5sN3B656qYV8JW7HzKzzsApwGcVGL+ISNWpWzesjVDS+gh798K6dUdfQbR2bZiK4s03w0prsWrVClcKffe78PvfV3jYpSZ6d883s5uBGYTLK59096VmNg7Ic/cpJXz8LGCcmR0EDgM3unsx0/OJiKSBBg3g1FPDozgFg8dFH4lOl11GWjNWRCQNlLRmrJa4ERFJc0r0IiJpToleRCTNKdGLiKQ5JXoRkTSnRC8ikuaU6EVE0pwSvYhImqt2N0yZ2Rbg83LsoiWwtYLCSaZ0OQ7QsVRH6XIcoGMpcKK7x10lvtol+vIys7zi7g5LJelyHKBjqY7S5ThAx5IIdd2IiKQ5JXoRkTSXjol+YrIDqCDpchygY6mO0uU4QMdSqrTroxcRkSOlY4teRERiKNGLiKS5lEz0ZjbczFaY2SozGxvn/Xpm9pfI+3PNrGPVR5mYBI7lajPbYmaLIo/rkhFnaczsSTPbbGYfFfO+mdkjkeNcbGY5VR1johI4liFmtiPmN7mrqmNMhJl1MLNZZvaxmS01s1vjlEmJ3yXBY0mV3yXTzOaZ2YeRY/lVnDIVm8PcPaUehOUMPwU6A3WBD4HsImV+CPxP5Plo4C/Jjrscx3I18Mdkx5rAsZwF5AAfFfP+CGA6YMAAYG6yYy7HsQwB/pbsOBM4juOBnMjzLOCTOP99pcTvkuCxpMrvYkCjyPM6wFxgQJEyFZrDUrFF3w9Y5e6fufsBYDIwskiZkcBTkecvAmebmVVhjIlK5FhSgrvPAUpaD3gk8LQH7wNNzez4qomubBI4lpTg7hvdfWHk+S5gGdCuSLGU+F0SPJaUEPlb7468rBN5FL0qpkJzWCom+nbA2pjX6zj6By8s4+75wA6gRZVEVzaJHAvApZHT6hfNrHJWD658iR5rqjgjcuo93cy6JTuY0kRO/fsQWo+xUu53KeFYIEV+FzPLMLNFwGbgDXcv9nepiByWiom+pnkN6OjuPYE3iNbykjwLCfOK9AIeBV5JcjwlMrNGwEvAbe6+M9nxlEcpx5Iyv4u7H3L33kB7oJ+Zda/M70vFRL8eiG3Vto9si1vGzGoDTYBtVRJd2ZR6LO6+zd2/ibx8AuhbRbFVtER+t5Tg7jsLTr3dfRpQx8xaJjmsuMysDiExPuvuf41TJGV+l9KOJZV+lwLuvh2YBQwv8laF5rBUTPTzgVPMrJOZ1SUMVEwpUmYK8P3I81HATI+MalQzpR5Lkf7Siwh9k6loCnBV5CqPAcAOd9+Y7KCOhZm1KegvNbN+hP+Pql1DIhLj/wOWufv4YoqlxO+SyLGk0O/SysyaRp7XB84FlhcpVqE5rPaxfjBZ3D3fzG4GZhCuWnnS3Zea2Tggz92nEP6D+LOZrSIMqo1OXsTFS/BYbjGzi4B8wrFcnbSAS2BmzxOuemhpZuuAuwmDTLj7/wDTCFd4rAL2AtckJ9LSJXAso4CbzCwf2AeMrqYNiYHA94Alkf5ggDuAEyDlfpdEjiVVfpfjgafMLINQGb3g7n+rzBymKRBERNJcKnbdiIhIGSjRi4ikOSV6EZE0p0QvIpLmlOhFRNKcEr2ISJpTohcRSXP/H/K2Hkipqsk2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = history.history['accuracy']\n",
    "val_acc = history.history['val_accuracy']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    " \n",
    "epochs = range(len(acc))\n",
    " \n",
    "plt.plot(epochs, acc, 'b', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'r', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    " \n",
    "plt.figure()\n",
    " \n",
    "plt.plot(epochs, loss, 'b', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    " \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "model.save(\"model.h5\")\n",
    "w2v_model.save('w2v_model.w2v')\n",
    "with open('tokenizer.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "with open('encoder.pickle', 'wb') as handle:\n",
    "    pickle.dump(encoder, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}