part_7_CNN_approach_transfer_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<!--NOTEBOOK_INFORMATION-->\n",
    "<img id=\"r-1060983\" data-claire-element-id=\"1061343\" src=\"http://www.siteduzero.com/favicon.ico\" alt=\"Image utilisateur\">\n",
    "    <p>\n",
    "        **<font color='#D2691E'size=\"6\">Image classification (7/9)</font>**.\n",
    "    </p>\n",
    "    <p>\n",
    "         This notebook discusses the classification of project images with the approach of convolution neuron networks.\n",
    "     It uses the <b> transfer learning </b> technique, implemented on the <b> VGG16 </b>, with <b> feature extraction from the convolution layers </b> and <b> fully connected layer training </b>.\n",
    "     </p>\n",
    "    \n",
    "<p>\n",
    "     N.B: It has been parameterized thanks to the work done on the notebooks provided in appendix\n",
    "     <b> Annex_1_CNN_approach_transfer_learning_over_parameters </b> and <b> Annex_2_CNN_approach_results_analyzis </b>\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    <center>\n",
    "        **<font color='\t#D2691E'size=\"6\">ROADMAP</font>**\n",
    "    </center>\n",
    "</p>\n",
    "<img align=\"left\" style=\"padding-right:10px;\" src=\"./images/part_7.jpg\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    <center>\n",
    "        **<font color='\t#D2691E'size=\"6\">PLAN</font>**\n",
    "    </center>\n",
    "</p>\n",
    "\n",
    "<p>\n",
    "        **<font color='#D2691E'size=\"4\">0) Libraries and functions import</font>**\n",
    "</p>\n",
    "<p>\n",
    "        **<font color='#D2691E'size=\"4\">I) CNN parameters calibration</font>**\n",
    "</p>\n",
    "<p>\n",
    "        **<font color='#D2691E'size=\"4\">II) Looping over the CNN parameters</font>**\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "        **<font color='#D2691E'size=\"4\">0) Libraries and functions import</font>**\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pickle \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import time\n",
    "\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.layers import Flatten,Dense,Dropout\n",
    "from keras.models import Model\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.optimizers import SGD, Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from context import datasources_path, pickles_path, temp_files_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functions_tailored import select_N_random_races\n",
    "from functions_tailored import build_train_validation_and_test_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "L_evaluation_cols = ['races',\n",
    "                     'training_len',\n",
    "                     'testing_len',\n",
    "                     'n_races',\n",
    "                     'batch_size',\n",
    "                     'learning_rate',\n",
    "                     'fitting_time',\n",
    "                     'prediction_time',\n",
    "                     'epochs_losses',\n",
    "                     'epochs_accuracies',\n",
    "                     'epochs_val_losses',\n",
    "                     'epochs_val_accuracies',\n",
    "                     'test_loss',\n",
    "                     'test_accuracy',\n",
    "                     'optimizer'\n",
    "                    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "RELOAD_EVALUATION = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>races</th>\n",
       "      <th>training_len</th>\n",
       "      <th>testing_len</th>\n",
       "      <th>n_races</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>fitting_time</th>\n",
       "      <th>prediction_time</th>\n",
       "      <th>epochs_losses</th>\n",
       "      <th>epochs_accuracies</th>\n",
       "      <th>epochs_val_losses</th>\n",
       "      <th>epochs_val_accuracies</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>optimizer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>['beagle', 'bernese_mountain_dog', 'dhole', 'e...</td>\n",
       "      <td>982</td>\n",
       "      <td>441</td>\n",
       "      <td>10</td>\n",
       "      <td>200</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>7584</td>\n",
       "      <td>109</td>\n",
       "      <td>[11.640229279543377, 8.675667757900571, 6.2707...</td>\n",
       "      <td>[0.13645621251064502, 0.320773934152607, 0.486...</td>\n",
       "      <td>[8.413873997266078, 6.248528911602387, 4.20287...</td>\n",
       "      <td>[0.29411764899643583, 0.43343653213867095, 0.5...</td>\n",
       "      <td>1.335279</td>\n",
       "      <td>0.868481</td>\n",
       "      <td>adam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>['beagle', 'bernese_mountain_dog', 'dhole', 'e...</td>\n",
       "      <td>982</td>\n",
       "      <td>441</td>\n",
       "      <td>10</td>\n",
       "      <td>200</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>21151</td>\n",
       "      <td>97</td>\n",
       "      <td>[13.164204929608191, 12.914754133360448, 12.32...</td>\n",
       "      <td>[0.10081466296479076, 0.11099796265727876, 0.1...</td>\n",
       "      <td>[12.512846996909694, 11.754830443084055, 10.95...</td>\n",
       "      <td>[0.10216718466935143, 0.12074303737734862, 0.1...</td>\n",
       "      <td>1.216085</td>\n",
       "      <td>0.854875</td>\n",
       "      <td>sgd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>['beagle', 'bernese_mountain_dog', 'dhole', 'e...</td>\n",
       "      <td>982</td>\n",
       "      <td>441</td>\n",
       "      <td>10</td>\n",
       "      <td>200</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>5452</td>\n",
       "      <td>97</td>\n",
       "      <td>[11.432109793917467, 6.4637340460192645, 3.881...</td>\n",
       "      <td>[0.16598777837156037, 0.4755600845012548, 0.67...</td>\n",
       "      <td>[8.568536076383324, 4.294536177218883, 3.03362...</td>\n",
       "      <td>[0.3188854513614908, 0.5975232375295538, 0.684...</td>\n",
       "      <td>1.159407</td>\n",
       "      <td>0.884354</td>\n",
       "      <td>adam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>['beagle', 'bernese_mountain_dog', 'dhole', 'e...</td>\n",
       "      <td>982</td>\n",
       "      <td>441</td>\n",
       "      <td>10</td>\n",
       "      <td>200</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>11787</td>\n",
       "      <td>98</td>\n",
       "      <td>[12.859470243123786, 11.982571741714011, 10.75...</td>\n",
       "      <td>[0.11608961356572367, 0.13849287499842478, 0.1...</td>\n",
       "      <td>[11.430217060880395, 9.780132278938412, 7.9990...</td>\n",
       "      <td>[0.12074303419412843, 0.18885448999449195, 0.2...</td>\n",
       "      <td>1.033548</td>\n",
       "      <td>0.877551</td>\n",
       "      <td>sgd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>['beagle', 'bernese_mountain_dog', 'dhole', 'e...</td>\n",
       "      <td>982</td>\n",
       "      <td>441</td>\n",
       "      <td>10</td>\n",
       "      <td>200</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>3344</td>\n",
       "      <td>97</td>\n",
       "      <td>[10.06785662256773, 4.840170306732116, 3.07574...</td>\n",
       "      <td>[0.2983706677275617, 0.6598778002859376, 0.790...</td>\n",
       "      <td>[5.666034890402212, 3.9021638406688584, 2.7525...</td>\n",
       "      <td>[0.5789473896425206, 0.7151702664584937, 0.798...</td>\n",
       "      <td>2.807766</td>\n",
       "      <td>0.807256</td>\n",
       "      <td>adam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>['beagle', 'bernese_mountain_dog', 'dhole', 'e...</td>\n",
       "      <td>982</td>\n",
       "      <td>441</td>\n",
       "      <td>10</td>\n",
       "      <td>200</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>7370</td>\n",
       "      <td>102</td>\n",
       "      <td>[11.965150679687376, 9.44600970196384, 6.06742...</td>\n",
       "      <td>[0.1395112039661699, 0.29633401464784703, 0.52...</td>\n",
       "      <td>[9.814261461559095, 6.352773416153049, 4.72292...</td>\n",
       "      <td>[0.24767801368568704, 0.49535603161566766, 0.6...</td>\n",
       "      <td>1.040372</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>sgd</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               races  training_len  \\\n",
       "0  ['beagle', 'bernese_mountain_dog', 'dhole', 'e...           982   \n",
       "1  ['beagle', 'bernese_mountain_dog', 'dhole', 'e...           982   \n",
       "2  ['beagle', 'bernese_mountain_dog', 'dhole', 'e...           982   \n",
       "3  ['beagle', 'bernese_mountain_dog', 'dhole', 'e...           982   \n",
       "4  ['beagle', 'bernese_mountain_dog', 'dhole', 'e...           982   \n",
       "5  ['beagle', 'bernese_mountain_dog', 'dhole', 'e...           982   \n",
       "\n",
       "   testing_len  n_races  batch_size  learning_rate  fitting_time  \\\n",
       "0          441       10         200       0.000008          7584   \n",
       "1          441       10         200       0.000008         21151   \n",
       "2          441       10         200       0.000019          5452   \n",
       "3          441       10         200       0.000019         11787   \n",
       "4          441       10         200       0.000076          3344   \n",
       "5          441       10         200       0.000076          7370   \n",
       "\n",
       "   prediction_time                                      epochs_losses  \\\n",
       "0              109  [11.640229279543377, 8.675667757900571, 6.2707...   \n",
       "1               97  [13.164204929608191, 12.914754133360448, 12.32...   \n",
       "2               97  [11.432109793917467, 6.4637340460192645, 3.881...   \n",
       "3               98  [12.859470243123786, 11.982571741714011, 10.75...   \n",
       "4               97  [10.06785662256773, 4.840170306732116, 3.07574...   \n",
       "5              102  [11.965150679687376, 9.44600970196384, 6.06742...   \n",
       "\n",
       "                                   epochs_accuracies  \\\n",
       "0  [0.13645621251064502, 0.320773934152607, 0.486...   \n",
       "1  [0.10081466296479076, 0.11099796265727876, 0.1...   \n",
       "2  [0.16598777837156037, 0.4755600845012548, 0.67...   \n",
       "3  [0.11608961356572367, 0.13849287499842478, 0.1...   \n",
       "4  [0.2983706677275617, 0.6598778002859376, 0.790...   \n",
       "5  [0.1395112039661699, 0.29633401464784703, 0.52...   \n",
       "\n",
       "                                   epochs_val_losses  \\\n",
       "0  [8.413873997266078, 6.248528911602387, 4.20287...   \n",
       "1  [12.512846996909694, 11.754830443084055, 10.95...   \n",
       "2  [8.568536076383324, 4.294536177218883, 3.03362...   \n",
       "3  [11.430217060880395, 9.780132278938412, 7.9990...   \n",
       "4  [5.666034890402212, 3.9021638406688584, 2.7525...   \n",
       "5  [9.814261461559095, 6.352773416153049, 4.72292...   \n",
       "\n",
       "                               epochs_val_accuracies  test_loss  \\\n",
       "0  [0.29411764899643583, 0.43343653213867095, 0.5...   1.335279   \n",
       "1  [0.10216718466935143, 0.12074303737734862, 0.1...   1.216085   \n",
       "2  [0.3188854513614908, 0.5975232375295538, 0.684...   1.159407   \n",
       "3  [0.12074303419412843, 0.18885448999449195, 0.2...   1.033548   \n",
       "4  [0.5789473896425206, 0.7151702664584937, 0.798...   2.807766   \n",
       "5  [0.24767801368568704, 0.49535603161566766, 0.6...   1.040372   \n",
       "\n",
       "   test_accuracy optimizer  \n",
       "0       0.868481      adam  \n",
       "1       0.854875       sgd  \n",
       "2       0.884354      adam  \n",
       "3       0.877551       sgd  \n",
       "4       0.807256      adam  \n",
       "5       0.897959       sgd  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if RELOAD_EVALUATION == True :\n",
    "    df_evaluation_final = pd.read_csv('df_evaluation_final.csv')\n",
    "    df_evaluation_final.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "\n",
    "elif RELOAD_EVALUATION == False :\n",
    "    df_evaluation_final = pd.DataFrame(columns = L_evaluation_cols)\n",
    "df_evaluation_final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "        **<font color='#D2691E'size=\"4\">I) CNN parameters calibration</font>**\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['beagle',\n",
       " 'bernese_mountain_dog',\n",
       " 'dhole',\n",
       " 'english_setter',\n",
       " 'japanese_spaniel',\n",
       " 'kelpie',\n",
       " 'labrador_retriever',\n",
       " 'rottweiler',\n",
       " 'siberian_husky',\n",
       " 'west_highland_white_terrier']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L_batch_sizes = [300,200]\n",
    "dict_lr_ranges = {\n",
    "    0:[5e-6,1e-5],\n",
    "    1:[1e-5,5e-5],\n",
    "    2:[5e-5,1e-4]\n",
    "}\n",
    "EPOCHS = 100 \n",
    "NN_CALLBACKS = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='auto')\n",
    "L_filtered_races = pickle.load(open(pickles_path+\"L_10_filtered_races.p\", \"rb\" ))\n",
    "RACE_NUMBER = len(L_filtered_races)\n",
    "L_filtered_races"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "        **<font color='#D2691E'size=\"4\">II) Looping over the CNN parameters</font>**\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filtered_races : ['beagle', 'bernese_mountain_dog', 'dhole', 'english_setter', 'japanese_spaniel', 'kelpie', 'labrador_retriever', 'rottweiler', 'siberian_husky', 'west_highland_white_terrier']\n",
      "\n",
      "building the train and test datasets ...\n",
      "\n",
      "looping over the vgg16 parameters ...\n",
      "\n",
      "\n",
      "\n",
      "#############################################################################################\n",
      "training the model with params :\n",
      "Batch size = 300 | Learning rate = 8.170208154307469e-06\n",
      "fitting the model ...\n",
      "optimizer : adam\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 555s 566ms/step - loss: 12.8392 - acc: 0.1110 - val_loss: 9.2915 - val_acc: 0.2446\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 544s 554ms/step - loss: 10.1730 - acc: 0.2464 - val_loss: 7.4397 - val_acc: 0.3591\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 525s 534ms/step - loss: 7.9755 - acc: 0.3686 - val_loss: 5.8755 - val_acc: 0.4644\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 541s 550ms/step - loss: 5.8038 - acc: 0.5010 - val_loss: 4.3003 - val_acc: 0.5635\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 527s 537ms/step - loss: 4.5094 - acc: 0.6161 - val_loss: 3.6455 - val_acc: 0.6440\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 540s 550ms/step - loss: 3.6673 - acc: 0.6833 - val_loss: 2.7963 - val_acc: 0.7152\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 2.6712 - acc: 0.7719 - val_loss: 2.5152 - val_acc: 0.7461\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 536s 546ms/step - loss: 2.5363 - acc: 0.7953 - val_loss: 2.4323 - val_acc: 0.7523\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 539s 549ms/step - loss: 2.3316 - acc: 0.8157 - val_loss: 2.3183 - val_acc: 0.7771\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 527s 536ms/step - loss: 1.9999 - acc: 0.8432 - val_loss: 2.2482 - val_acc: 0.7864\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 534s 544ms/step - loss: 1.8079 - acc: 0.8635 - val_loss: 2.2394 - val_acc: 0.7864\n",
      "Epoch 12/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 1.7033 - acc: 0.8758 - val_loss: 2.2351 - val_acc: 0.7988\n",
      "Epoch 13/100\n",
      "982/982 [==============================] - 542s 552ms/step - loss: 1.6705 - acc: 0.8758 - val_loss: 2.1787 - val_acc: 0.8080\n",
      "Epoch 14/100\n",
      "982/982 [==============================] - 538s 547ms/step - loss: 1.6004 - acc: 0.8778 - val_loss: 2.0171 - val_acc: 0.8142\n",
      "Epoch 15/100\n",
      "982/982 [==============================] - 545s 555ms/step - loss: 1.4973 - acc: 0.8931 - val_loss: 1.8211 - val_acc: 0.8266\n",
      "Epoch 16/100\n",
      "982/982 [==============================] - 544s 554ms/step - loss: 1.3455 - acc: 0.8971 - val_loss: 1.5166 - val_acc: 0.8483\n",
      "Epoch 17/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 1.0730 - acc: 0.9114 - val_loss: 1.2673 - val_acc: 0.8328\n",
      "Epoch 18/100\n",
      "982/982 [==============================] - 538s 548ms/step - loss: 0.7453 - acc: 0.9196 - val_loss: 2.1094 - val_acc: 0.7802\n",
      "Epoch 19/100\n",
      "982/982 [==============================] - 539s 549ms/step - loss: 0.7400 - acc: 0.9165 - val_loss: 2.3040 - val_acc: 0.7616\n",
      "Epoch 20/100\n",
      "982/982 [==============================] - 536s 546ms/step - loss: 0.5476 - acc: 0.9287 - val_loss: 1.6130 - val_acc: 0.8204\n",
      "fitting_time : 10754 seconds\n",
      "\n",
      "computing the accuracy on the test data ...\n",
      "\n",
      "test_loss : 1.5964049271132603 | test_accuracy : 0.8299319727891157\n",
      "prediction_time : 98 seconds\n",
      "\n",
      "CSV written !\n",
      "\n",
      "training_time : 10853.730961561203 seconds\n",
      "\n",
      "fitting the model ...\n",
      "optimizer : sgd\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 546s 556ms/step - loss: 12.9565 - acc: 0.0967 - val_loss: 11.4532 - val_acc: 0.1084\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 532s 542ms/step - loss: 12.6666 - acc: 0.0998 - val_loss: 10.9037 - val_acc: 0.1207\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 517s 526ms/step - loss: 12.1934 - acc: 0.1191 - val_loss: 10.1279 - val_acc: 0.1579\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 11.6922 - acc: 0.1375 - val_loss: 9.3096 - val_acc: 0.1889\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 543s 553ms/step - loss: 10.9591 - acc: 0.1680 - val_loss: 8.5788 - val_acc: 0.2508\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 520s 530ms/step - loss: 10.4804 - acc: 0.2189 - val_loss: 7.9002 - val_acc: 0.2972\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 526s 536ms/step - loss: 9.4942 - acc: 0.2464 - val_loss: 7.2403 - val_acc: 0.3622\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 541s 551ms/step - loss: 9.1430 - acc: 0.2892 - val_loss: 6.6254 - val_acc: 0.4149\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 521s 531ms/step - loss: 8.1765 - acc: 0.3310 - val_loss: 6.0650 - val_acc: 0.4613\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 544s 554ms/step - loss: 7.7232 - acc: 0.3921 - val_loss: 5.5783 - val_acc: 0.5139\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 550s 560ms/step - loss: 7.0303 - acc: 0.4206 - val_loss: 5.1905 - val_acc: 0.5666\n",
      "Epoch 12/100\n",
      "982/982 [==============================] - 530s 540ms/step - loss: 6.6358 - acc: 0.4450 - val_loss: 4.8821 - val_acc: 0.5851\n",
      "Epoch 13/100\n",
      "982/982 [==============================] - 531s 540ms/step - loss: 6.0337 - acc: 0.5061 - val_loss: 4.6550 - val_acc: 0.6068\n",
      "Epoch 14/100\n",
      "982/982 [==============================] - 531s 541ms/step - loss: 5.8300 - acc: 0.5092 - val_loss: 4.4672 - val_acc: 0.6192\n",
      "Epoch 15/100\n",
      "982/982 [==============================] - 540s 550ms/step - loss: 5.4355 - acc: 0.5367 - val_loss: 4.2624 - val_acc: 0.6347\n",
      "Epoch 16/100\n",
      "982/982 [==============================] - 535s 544ms/step - loss: 5.2744 - acc: 0.5499 - val_loss: 4.0173 - val_acc: 0.6440\n",
      "Epoch 17/100\n",
      "982/982 [==============================] - 530s 540ms/step - loss: 4.8187 - acc: 0.5723 - val_loss: 3.7361 - val_acc: 0.6687\n",
      "Epoch 18/100\n",
      "982/982 [==============================] - 535s 545ms/step - loss: 4.4862 - acc: 0.6018 - val_loss: 3.4451 - val_acc: 0.6533\n",
      "Epoch 19/100\n",
      "982/982 [==============================] - 532s 541ms/step - loss: 4.2936 - acc: 0.6110 - val_loss: 3.1808 - val_acc: 0.6656\n",
      "Epoch 20/100\n",
      "982/982 [==============================] - 523s 532ms/step - loss: 4.1636 - acc: 0.6314 - val_loss: 2.8848 - val_acc: 0.6749\n",
      "Epoch 21/100\n",
      "982/982 [==============================] - 539s 549ms/step - loss: 3.8740 - acc: 0.6365 - val_loss: 2.5934 - val_acc: 0.7059\n",
      "Epoch 22/100\n",
      "982/982 [==============================] - 546s 556ms/step - loss: 3.4692 - acc: 0.6660 - val_loss: 2.3471 - val_acc: 0.7276\n",
      "Epoch 23/100\n",
      "982/982 [==============================] - 523s 532ms/step - loss: 3.2116 - acc: 0.6762 - val_loss: 2.1628 - val_acc: 0.7399\n",
      "Epoch 24/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 2.9177 - acc: 0.6945 - val_loss: 2.0549 - val_acc: 0.7585\n",
      "Epoch 25/100\n",
      "982/982 [==============================] - 527s 537ms/step - loss: 2.8941 - acc: 0.7118 - val_loss: 1.9329 - val_acc: 0.7616\n",
      "Epoch 26/100\n",
      "982/982 [==============================] - 522s 532ms/step - loss: 2.7658 - acc: 0.7251 - val_loss: 1.8487 - val_acc: 0.7709\n",
      "Epoch 27/100\n",
      "982/982 [==============================] - 526s 536ms/step - loss: 2.4570 - acc: 0.7363 - val_loss: 1.7866 - val_acc: 0.7678\n",
      "Epoch 28/100\n",
      "982/982 [==============================] - 534s 544ms/step - loss: 2.4293 - acc: 0.7464 - val_loss: 1.6926 - val_acc: 0.7833\n",
      "Epoch 29/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 1.9409 - acc: 0.7892 - val_loss: 1.6073 - val_acc: 0.7864\n",
      "Epoch 30/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 1.9345 - acc: 0.7770 - val_loss: 1.5145 - val_acc: 0.8050\n",
      "Epoch 31/100\n",
      "982/982 [==============================] - 548s 558ms/step - loss: 1.7960 - acc: 0.7872 - val_loss: 1.4458 - val_acc: 0.8111\n",
      "Epoch 32/100\n",
      "982/982 [==============================] - 536s 546ms/step - loss: 1.8766 - acc: 0.7912 - val_loss: 1.4305 - val_acc: 0.8111\n",
      "Epoch 33/100\n",
      "982/982 [==============================] - 536s 546ms/step - loss: 1.8020 - acc: 0.7811 - val_loss: 1.4078 - val_acc: 0.8235\n",
      "Epoch 34/100\n",
      "982/982 [==============================] - 522s 531ms/step - loss: 1.7190 - acc: 0.8310 - val_loss: 1.3449 - val_acc: 0.8204\n",
      "Epoch 35/100\n",
      "982/982 [==============================] - 524s 534ms/step - loss: 1.7142 - acc: 0.8198 - val_loss: 1.2813 - val_acc: 0.8297\n",
      "Epoch 36/100\n",
      "982/982 [==============================] - 534s 544ms/step - loss: 1.5513 - acc: 0.8208 - val_loss: 1.2379 - val_acc: 0.8328\n",
      "Epoch 37/100\n",
      "982/982 [==============================] - 514s 523ms/step - loss: 1.4393 - acc: 0.8422 - val_loss: 1.2115 - val_acc: 0.8328\n",
      "Epoch 38/100\n",
      "982/982 [==============================] - 522s 531ms/step - loss: 1.3051 - acc: 0.8493 - val_loss: 1.1937 - val_acc: 0.8359\n",
      "Epoch 39/100\n",
      "982/982 [==============================] - 530s 540ms/step - loss: 1.2468 - acc: 0.8544 - val_loss: 1.1821 - val_acc: 0.8390\n",
      "Epoch 40/100\n",
      "982/982 [==============================] - 521s 531ms/step - loss: 1.2506 - acc: 0.8585 - val_loss: 1.1861 - val_acc: 0.8452\n",
      "Epoch 41/100\n",
      "982/982 [==============================] - 521s 530ms/step - loss: 1.2156 - acc: 0.8574 - val_loss: 1.1884 - val_acc: 0.8390\n",
      "Epoch 42/100\n",
      "982/982 [==============================] - 512s 522ms/step - loss: 1.1463 - acc: 0.8605 - val_loss: 1.1809 - val_acc: 0.8421\n",
      "Epoch 43/100\n",
      "982/982 [==============================] - 552s 562ms/step - loss: 1.0339 - acc: 0.8747 - val_loss: 1.1571 - val_acc: 0.8483\n",
      "Epoch 44/100\n",
      "982/982 [==============================] - 523s 533ms/step - loss: 1.0718 - acc: 0.8717 - val_loss: 1.1312 - val_acc: 0.8514\n",
      "Epoch 45/100\n",
      "982/982 [==============================] - 522s 532ms/step - loss: 1.1301 - acc: 0.8656 - val_loss: 1.0881 - val_acc: 0.8514\n",
      "Epoch 46/100\n",
      "982/982 [==============================] - 524s 534ms/step - loss: 0.8062 - acc: 0.8961 - val_loss: 1.0459 - val_acc: 0.8545\n",
      "Epoch 47/100\n",
      "982/982 [==============================] - 534s 543ms/step - loss: 0.9808 - acc: 0.8839 - val_loss: 1.0109 - val_acc: 0.8545\n",
      "Epoch 48/100\n",
      "982/982 [==============================] - 516s 525ms/step - loss: 0.8229 - acc: 0.8951 - val_loss: 0.9828 - val_acc: 0.8576\n",
      "Epoch 49/100\n",
      "982/982 [==============================] - 531s 540ms/step - loss: 0.7421 - acc: 0.9033 - val_loss: 0.9643 - val_acc: 0.8576\n",
      "Epoch 50/100\n",
      "982/982 [==============================] - 531s 541ms/step - loss: 0.6800 - acc: 0.9155 - val_loss: 0.9556 - val_acc: 0.8607\n",
      "Epoch 51/100\n",
      "982/982 [==============================] - 533s 542ms/step - loss: 0.6156 - acc: 0.9084 - val_loss: 0.9492 - val_acc: 0.8607\n",
      "Epoch 52/100\n",
      "982/982 [==============================] - 522s 532ms/step - loss: 0.7200 - acc: 0.9002 - val_loss: 0.9415 - val_acc: 0.8607\n",
      "Epoch 53/100\n",
      "982/982 [==============================] - 527s 537ms/step - loss: 0.7655 - acc: 0.9053 - val_loss: 0.9351 - val_acc: 0.8607\n",
      "Epoch 54/100\n",
      "982/982 [==============================] - 511s 520ms/step - loss: 0.8441 - acc: 0.8982 - val_loss: 0.9251 - val_acc: 0.8607\n",
      "Epoch 55/100\n",
      "982/982 [==============================] - 514s 524ms/step - loss: 0.6429 - acc: 0.9165 - val_loss: 0.9101 - val_acc: 0.8638\n",
      "Epoch 56/100\n",
      "982/982 [==============================] - 528s 538ms/step - loss: 0.7959 - acc: 0.9012 - val_loss: 0.9032 - val_acc: 0.8638\n",
      "Epoch 57/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 0.6008 - acc: 0.9155 - val_loss: 0.9040 - val_acc: 0.8669\n",
      "Epoch 58/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 0.6019 - acc: 0.9216 - val_loss: 0.9101 - val_acc: 0.8700\n",
      "Epoch 59/100\n",
      "982/982 [==============================] - 526s 536ms/step - loss: 0.6503 - acc: 0.9175 - val_loss: 0.9173 - val_acc: 0.8731\n",
      "fitting_time : 31265 seconds\n",
      "\n",
      "computing the accuracy on the test data ...\n",
      "\n",
      "test_loss : 1.0023600548303047 | test_accuracy : 0.8662131507110163\n",
      "prediction_time : 97 seconds\n",
      "\n",
      "CSV written !\n",
      "\n",
      "training_time : 42218.79366469383 seconds\n",
      "\n",
      "\n",
      "\n",
      "#############################################################################################\n",
      "training the model with params :\n",
      "Batch size = 300 | Learning rate = 4.968027319721656e-05\n",
      "fitting the model ...\n",
      "optimizer : adam\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 552s 562ms/step - loss: 11.0548 - acc: 0.2179 - val_loss: 5.4229 - val_acc: 0.5325\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 544s 554ms/step - loss: 4.5967 - acc: 0.6181 - val_loss: 2.6057 - val_acc: 0.7368\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 535s 545ms/step - loss: 1.9466 - acc: 0.8259 - val_loss: 1.8214 - val_acc: 0.8173\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 515s 525ms/step - loss: 0.8593 - acc: 0.9287 - val_loss: 1.5645 - val_acc: 0.8545\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 516s 525ms/step - loss: 0.6205 - acc: 0.9409 - val_loss: 1.1835 - val_acc: 0.8793\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 532s 542ms/step - loss: 0.4964 - acc: 0.9603 - val_loss: 1.0666 - val_acc: 0.9071\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 543s 553ms/step - loss: 0.4359 - acc: 0.9593 - val_loss: 1.1574 - val_acc: 0.9009\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 535s 545ms/step - loss: 0.2935 - acc: 0.9725 - val_loss: 1.0129 - val_acc: 0.9164\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 546s 556ms/step - loss: 0.2616 - acc: 0.9786 - val_loss: 1.1793 - val_acc: 0.8978\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 541s 551ms/step - loss: 0.2283 - acc: 0.9817 - val_loss: 1.2478 - val_acc: 0.8885\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 546s 556ms/step - loss: 0.1669 - acc: 0.9837 - val_loss: 1.2881 - val_acc: 0.8854\n",
      "fitting_time : 5905 seconds\n",
      "\n",
      "computing the accuracy on the test data ...\n",
      "\n",
      "test_loss : 1.1000725590452856 | test_accuracy : 0.9002267573696145\n",
      "prediction_time : 98 seconds\n",
      "\n",
      "CSV written !\n",
      "\n",
      "training_time : 6006.012809753418 seconds\n",
      "\n",
      "fitting the model ...\n",
      "optimizer : sgd\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 523s 533ms/step - loss: 12.7519 - acc: 0.1029 - val_loss: 11.1264 - val_acc: 0.1362\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 529s 539ms/step - loss: 11.5853 - acc: 0.1527 - val_loss: 8.7815 - val_acc: 0.2260\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 535s 545ms/step - loss: 9.8355 - acc: 0.2413 - val_loss: 7.3063 - val_acc: 0.3963\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 529s 539ms/step - loss: 7.3274 - acc: 0.4053 - val_loss: 4.4555 - val_acc: 0.5573\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 526s 536ms/step - loss: 5.1963 - acc: 0.5601 - val_loss: 3.2408 - val_acc: 0.6533\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 531s 541ms/step - loss: 3.9396 - acc: 0.6507 - val_loss: 2.0353 - val_acc: 0.7523\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 2.4331 - acc: 0.7637 - val_loss: 1.7717 - val_acc: 0.7957\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 519s 528ms/step - loss: 1.6875 - acc: 0.8259 - val_loss: 1.3441 - val_acc: 0.8328\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 528s 538ms/step - loss: 1.2593 - acc: 0.8666 - val_loss: 1.1577 - val_acc: 0.8576\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 526s 535ms/step - loss: 1.0693 - acc: 0.8890 - val_loss: 1.0494 - val_acc: 0.8854\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 532s 542ms/step - loss: 0.9114 - acc: 0.9104 - val_loss: 0.9573 - val_acc: 0.8885\n",
      "Epoch 12/100\n",
      "982/982 [==============================] - 530s 540ms/step - loss: 0.7030 - acc: 0.9246 - val_loss: 0.9145 - val_acc: 0.8916\n",
      "Epoch 13/100\n",
      "982/982 [==============================] - 541s 551ms/step - loss: 0.6105 - acc: 0.9318 - val_loss: 0.8735 - val_acc: 0.8854\n",
      "Epoch 14/100\n",
      "982/982 [==============================] - 534s 544ms/step - loss: 0.4707 - acc: 0.9450 - val_loss: 0.8451 - val_acc: 0.9071\n",
      "Epoch 15/100\n",
      "982/982 [==============================] - 534s 544ms/step - loss: 0.5233 - acc: 0.9470 - val_loss: 0.8673 - val_acc: 0.8978\n",
      "Epoch 16/100\n",
      "982/982 [==============================] - 525s 534ms/step - loss: 0.3110 - acc: 0.9593 - val_loss: 0.8397 - val_acc: 0.9071\n",
      "Epoch 17/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "982/982 [==============================] - 525s 534ms/step - loss: 0.3388 - acc: 0.9603 - val_loss: 0.7736 - val_acc: 0.9071\n",
      "Epoch 18/100\n",
      "982/982 [==============================] - 533s 542ms/step - loss: 0.2956 - acc: 0.9562 - val_loss: 0.7273 - val_acc: 0.9195\n",
      "Epoch 19/100\n",
      "982/982 [==============================] - 527s 537ms/step - loss: 0.2023 - acc: 0.9776 - val_loss: 0.7435 - val_acc: 0.9226\n",
      "Epoch 20/100\n",
      "982/982 [==============================] - 536s 546ms/step - loss: 0.2686 - acc: 0.9654 - val_loss: 0.7210 - val_acc: 0.9257\n",
      "Epoch 21/100\n",
      "982/982 [==============================] - 525s 535ms/step - loss: 0.1689 - acc: 0.9725 - val_loss: 0.6995 - val_acc: 0.9226\n",
      "Epoch 22/100\n",
      "982/982 [==============================] - 527s 537ms/step - loss: 0.1503 - acc: 0.9827 - val_loss: 0.6903 - val_acc: 0.9226\n",
      "Epoch 23/100\n",
      "982/982 [==============================] - 542s 552ms/step - loss: 0.1467 - acc: 0.9817 - val_loss: 0.6892 - val_acc: 0.9040\n",
      "Epoch 24/100\n",
      "982/982 [==============================] - 531s 541ms/step - loss: 0.1478 - acc: 0.9766 - val_loss: 0.6924 - val_acc: 0.8885\n",
      "Epoch 25/100\n",
      "982/982 [==============================] - 527s 536ms/step - loss: 0.1106 - acc: 0.9837 - val_loss: 0.6733 - val_acc: 0.8916\n",
      "Epoch 26/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 0.1328 - acc: 0.9776 - val_loss: 0.6684 - val_acc: 0.9040\n",
      "Epoch 27/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 0.1010 - acc: 0.9878 - val_loss: 0.6635 - val_acc: 0.9071\n",
      "Epoch 28/100\n",
      "982/982 [==============================] - 519s 529ms/step - loss: 0.1005 - acc: 0.9817 - val_loss: 0.6633 - val_acc: 0.9071\n",
      "Epoch 29/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 0.1018 - acc: 0.9786 - val_loss: 0.6651 - val_acc: 0.9009\n",
      "Epoch 30/100\n",
      "982/982 [==============================] - 526s 535ms/step - loss: 0.1168 - acc: 0.9796 - val_loss: 0.6620 - val_acc: 0.9133\n",
      "Epoch 31/100\n",
      "982/982 [==============================] - 533s 543ms/step - loss: 0.1208 - acc: 0.9837 - val_loss: 0.6538 - val_acc: 0.9164\n",
      "Epoch 32/100\n",
      "982/982 [==============================] - 528s 538ms/step - loss: 0.0368 - acc: 0.9908 - val_loss: 0.6428 - val_acc: 0.9226\n",
      "Epoch 33/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 0.0540 - acc: 0.9878 - val_loss: 0.6255 - val_acc: 0.9226\n",
      "Epoch 34/100\n",
      "982/982 [==============================] - 539s 549ms/step - loss: 0.0307 - acc: 0.9929 - val_loss: 0.6127 - val_acc: 0.9288\n",
      "Epoch 35/100\n",
      "982/982 [==============================] - 532s 542ms/step - loss: 0.0899 - acc: 0.9857 - val_loss: 0.6163 - val_acc: 0.9288\n",
      "Epoch 36/100\n",
      "982/982 [==============================] - 521s 531ms/step - loss: 0.0497 - acc: 0.9898 - val_loss: 0.6102 - val_acc: 0.9226\n",
      "Epoch 37/100\n",
      "982/982 [==============================] - 523s 532ms/step - loss: 0.0299 - acc: 0.9939 - val_loss: 0.5986 - val_acc: 0.9164\n",
      "Epoch 38/100\n",
      "982/982 [==============================] - 531s 540ms/step - loss: 0.0427 - acc: 0.9919 - val_loss: 0.6054 - val_acc: 0.9164\n",
      "Epoch 39/100\n",
      "982/982 [==============================] - 528s 537ms/step - loss: 0.0296 - acc: 0.9929 - val_loss: 0.6141 - val_acc: 0.9164\n",
      "Epoch 40/100\n",
      "982/982 [==============================] - 527s 537ms/step - loss: 0.0527 - acc: 0.9888 - val_loss: 0.6137 - val_acc: 0.9164\n",
      "fitting_time : 21204 seconds\n",
      "\n",
      "computing the accuracy on the test data ...\n",
      "\n",
      "test_loss : 0.8384133109413634 | test_accuracy : 0.9002267561531931\n",
      "prediction_time : 98 seconds\n",
      "\n",
      "CSV written !\n",
      "\n",
      "training_time : 27311.47902393341 seconds\n",
      "\n",
      "\n",
      "\n",
      "#############################################################################################\n",
      "training the model with params :\n",
      "Batch size = 300 | Learning rate = 5.890056624677413e-05\n",
      "fitting the model ...\n",
      "optimizer : adam\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 545s 555ms/step - loss: 11.8260 - acc: 0.1945 - val_loss: 7.5737 - val_acc: 0.4706\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 541s 551ms/step - loss: 7.1248 - acc: 0.5132 - val_loss: 6.6334 - val_acc: 0.5418\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 536s 545ms/step - loss: 5.8196 - acc: 0.6079 - val_loss: 6.0665 - val_acc: 0.5697\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 532s 541ms/step - loss: 4.8720 - acc: 0.6782 - val_loss: 5.1894 - val_acc: 0.6440\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 542s 552ms/step - loss: 4.1765 - acc: 0.7230 - val_loss: 4.2242 - val_acc: 0.6966\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 543s 553ms/step - loss: 2.9686 - acc: 0.7953 - val_loss: 3.8977 - val_acc: 0.7276\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 540s 550ms/step - loss: 2.6692 - acc: 0.8208 - val_loss: 3.6764 - val_acc: 0.7276\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 544s 554ms/step - loss: 2.5712 - acc: 0.8269 - val_loss: 3.3690 - val_acc: 0.7554\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 545s 555ms/step - loss: 2.0159 - acc: 0.8585 - val_loss: 1.6494 - val_acc: 0.8607\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 540s 550ms/step - loss: 0.7799 - acc: 0.9348 - val_loss: 2.8466 - val_acc: 0.7833\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 532s 541ms/step - loss: 0.5541 - acc: 0.9521 - val_loss: 1.4774 - val_acc: 0.8793\n",
      "Epoch 12/100\n",
      "982/982 [==============================] - 531s 540ms/step - loss: 0.2999 - acc: 0.9756 - val_loss: 1.2457 - val_acc: 0.8916\n",
      "Epoch 13/100\n",
      "982/982 [==============================] - 534s 544ms/step - loss: 0.3305 - acc: 0.9745 - val_loss: 1.2614 - val_acc: 0.8916\n",
      "Epoch 14/100\n",
      "982/982 [==============================] - 550s 560ms/step - loss: 0.3188 - acc: 0.9735 - val_loss: 0.9379 - val_acc: 0.9226\n",
      "Epoch 15/100\n",
      "982/982 [==============================] - 546s 556ms/step - loss: 0.1869 - acc: 0.9847 - val_loss: 0.8338 - val_acc: 0.9226\n",
      "Epoch 16/100\n",
      "982/982 [==============================] - 536s 546ms/step - loss: 0.1768 - acc: 0.9868 - val_loss: 0.8932 - val_acc: 0.9257\n",
      "Epoch 17/100\n",
      "982/982 [==============================] - 529s 539ms/step - loss: 0.1360 - acc: 0.9898 - val_loss: 0.8292 - val_acc: 0.9288\n",
      "Epoch 18/100\n",
      "982/982 [==============================] - 566s 576ms/step - loss: 0.1482 - acc: 0.9908 - val_loss: 1.0399 - val_acc: 0.9071\n",
      "Epoch 19/100\n",
      "982/982 [==============================] - 553s 564ms/step - loss: 0.1883 - acc: 0.9857 - val_loss: 0.9951 - val_acc: 0.9164\n",
      "Epoch 20/100\n",
      "982/982 [==============================] - 547s 557ms/step - loss: 0.0832 - acc: 0.9939 - val_loss: 1.3314 - val_acc: 0.8885\n",
      "fitting_time : 10832 seconds\n",
      "\n",
      "computing the accuracy on the test data ...\n",
      "\n",
      "test_loss : 1.56189243390717 | test_accuracy : 0.8820861678004536\n",
      "prediction_time : 98 seconds\n",
      "\n",
      "CSV written !\n",
      "\n",
      "training_time : 10933.23710513115 seconds\n",
      "\n",
      "fitting the model ...\n",
      "optimizer : sgd\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 537s 547ms/step - loss: 12.6420 - acc: 0.0927 - val_loss: 10.1341 - val_acc: 0.1858\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 521s 530ms/step - loss: 11.1217 - acc: 0.1762 - val_loss: 7.9288 - val_acc: 0.2941\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 540s 550ms/step - loss: 8.5286 - acc: 0.3136 - val_loss: 5.1026 - val_acc: 0.4923\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 545s 555ms/step - loss: 6.1383 - acc: 0.4796 - val_loss: 3.0030 - val_acc: 0.6718\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 535s 545ms/step - loss: 3.8254 - acc: 0.6354 - val_loss: 2.1336 - val_acc: 0.7492\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 529s 539ms/step - loss: 2.7645 - acc: 0.7434 - val_loss: 1.7018 - val_acc: 0.8173\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 525s 534ms/step - loss: 1.8017 - acc: 0.8075 - val_loss: 1.1514 - val_acc: 0.8638\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 534s 543ms/step - loss: 1.4157 - acc: 0.8554 - val_loss: 0.9278 - val_acc: 0.8824\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 526s 536ms/step - loss: 1.0711 - acc: 0.8992 - val_loss: 0.8541 - val_acc: 0.9040\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 528s 538ms/step - loss: 0.8421 - acc: 0.9124 - val_loss: 0.9204 - val_acc: 0.9102\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 535s 545ms/step - loss: 0.6895 - acc: 0.9297 - val_loss: 0.9785 - val_acc: 0.8947\n",
      "Epoch 12/100\n",
      "982/982 [==============================] - 556s 567ms/step - loss: 0.6582 - acc: 0.9308 - val_loss: 1.0291 - val_acc: 0.8854\n",
      "fitting_time : 6412 seconds\n",
      "\n",
      "computing the accuracy on the test data ...\n",
      "\n",
      "test_loss : 0.9866808197397582 | test_accuracy : 0.8820861678004536\n",
      "prediction_time : 98 seconds\n",
      "\n",
      "CSV written !\n",
      "\n",
      "training_time : 17445.289212942123 seconds\n",
      "\n",
      "\n",
      "\n",
      "#############################################################################################\n",
      "training the model with params :\n",
      "Batch size = 200 | Learning rate = 5.214164316844171e-06\n",
      "fitting the model ...\n",
      "optimizer : adam\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 472s 481ms/step - loss: 12.4543 - acc: 0.1202 - val_loss: 10.4411 - val_acc: 0.2291\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 423s 430ms/step - loss: 10.4954 - acc: 0.2434 - val_loss: 8.9770 - val_acc: 0.3282\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 426s 434ms/step - loss: 9.0998 - acc: 0.3289 - val_loss: 7.5671 - val_acc: 0.3994\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 407s 414ms/step - loss: 7.5611 - acc: 0.4185 - val_loss: 6.1408 - val_acc: 0.4675\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 423s 431ms/step - loss: 6.5826 - acc: 0.4664 - val_loss: 5.1655 - val_acc: 0.5108\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 489s 498ms/step - loss: 5.2101 - acc: 0.5570 - val_loss: 3.9605 - val_acc: 0.5944\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 414s 421ms/step - loss: 3.8617 - acc: 0.6385 - val_loss: 3.2118 - val_acc: 0.6471\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 421s 429ms/step - loss: 3.1869 - acc: 0.7128 - val_loss: 2.7979 - val_acc: 0.6966\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 369s 375ms/step - loss: 2.4588 - acc: 0.7424 - val_loss: 2.2882 - val_acc: 0.7214\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 390s 397ms/step - loss: 1.9497 - acc: 0.8086 - val_loss: 1.9465 - val_acc: 0.7616\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 393s 401ms/step - loss: 1.4862 - acc: 0.8462 - val_loss: 1.8040 - val_acc: 0.7802\n",
      "Epoch 12/100\n",
      "982/982 [==============================] - 423s 430ms/step - loss: 1.2058 - acc: 0.8646 - val_loss: 1.6548 - val_acc: 0.7864\n",
      "Epoch 13/100\n",
      "982/982 [==============================] - 404s 412ms/step - loss: 0.9889 - acc: 0.8931 - val_loss: 1.4747 - val_acc: 0.8080\n",
      "Epoch 14/100\n",
      "982/982 [==============================] - 335s 341ms/step - loss: 0.6998 - acc: 0.9104 - val_loss: 1.3931 - val_acc: 0.8142\n",
      "Epoch 15/100\n",
      "982/982 [==============================] - 405s 413ms/step - loss: 0.6082 - acc: 0.9277 - val_loss: 1.3466 - val_acc: 0.8204\n",
      "Epoch 16/100\n",
      "982/982 [==============================] - 453s 461ms/step - loss: 0.5149 - acc: 0.9379 - val_loss: 1.3157 - val_acc: 0.8390\n",
      "Epoch 17/100\n",
      "982/982 [==============================] - 429s 436ms/step - loss: 0.5050 - acc: 0.9379 - val_loss: 1.2900 - val_acc: 0.8452\n",
      "Epoch 18/100\n",
      "982/982 [==============================] - 468s 476ms/step - loss: 0.4425 - acc: 0.9440 - val_loss: 1.2646 - val_acc: 0.8483\n",
      "Epoch 19/100\n",
      "982/982 [==============================] - 336s 342ms/step - loss: 0.2636 - acc: 0.9644 - val_loss: 1.2834 - val_acc: 0.8514\n",
      "Epoch 20/100\n",
      "982/982 [==============================] - 397s 405ms/step - loss: 0.3500 - acc: 0.9603 - val_loss: 1.2529 - val_acc: 0.8514\n",
      "Epoch 21/100\n",
      "982/982 [==============================] - 420s 427ms/step - loss: 0.2613 - acc: 0.9633 - val_loss: 1.1885 - val_acc: 0.8514\n",
      "Epoch 22/100\n",
      "982/982 [==============================] - 428s 436ms/step - loss: 0.1904 - acc: 0.9674 - val_loss: 1.1658 - val_acc: 0.8607\n",
      "Epoch 23/100\n",
      "982/982 [==============================] - 418s 426ms/step - loss: 0.2459 - acc: 0.9654 - val_loss: 1.1420 - val_acc: 0.8607\n",
      "Epoch 24/100\n",
      "982/982 [==============================] - 419s 427ms/step - loss: 0.1431 - acc: 0.9807 - val_loss: 1.1273 - val_acc: 0.8638\n",
      "Epoch 25/100\n",
      "982/982 [==============================] - 408s 415ms/step - loss: 0.1803 - acc: 0.9715 - val_loss: 1.1432 - val_acc: 0.8607\n",
      "Epoch 26/100\n",
      "982/982 [==============================] - 418s 426ms/step - loss: 0.2015 - acc: 0.9776 - val_loss: 1.1576 - val_acc: 0.8607\n",
      "Epoch 27/100\n",
      "982/982 [==============================] - 397s 404ms/step - loss: 0.1040 - acc: 0.9857 - val_loss: 1.1452 - val_acc: 0.8576\n",
      "fitting_time : 11184 seconds\n",
      "\n",
      "computing the accuracy on the test data ...\n",
      "\n",
      "test_loss : 1.3905291066435324 | test_accuracy : 0.8503401348380004\n",
      "prediction_time : 97 seconds\n",
      "\n",
      "CSV written !\n",
      "\n",
      "training_time : 11284.241225481033 seconds\n",
      "\n",
      "fitting the model ...\n",
      "optimizer : sgd\n",
      "\n",
      "Train on 982 samples, validate on 323 samples\n",
      "Epoch 1/100\n",
      "982/982 [==============================] - 482s 491ms/step - loss: 12.4932 - acc: 0.1100 - val_loss: 11.6348 - val_acc: 0.1022\n",
      "Epoch 2/100\n",
      "982/982 [==============================] - 389s 396ms/step - loss: 12.3511 - acc: 0.1069 - val_loss: 11.1595 - val_acc: 0.1053\n",
      "Epoch 3/100\n",
      "982/982 [==============================] - 379s 386ms/step - loss: 11.9287 - acc: 0.1232 - val_loss: 10.5215 - val_acc: 0.1455\n",
      "Epoch 4/100\n",
      "982/982 [==============================] - 366s 373ms/step - loss: 11.5199 - acc: 0.1405 - val_loss: 10.0000 - val_acc: 0.1734\n",
      "Epoch 5/100\n",
      "982/982 [==============================] - 420s 428ms/step - loss: 11.2078 - acc: 0.1802 - val_loss: 9.4235 - val_acc: 0.1858\n",
      "Epoch 6/100\n",
      "982/982 [==============================] - 384s 391ms/step - loss: 10.5361 - acc: 0.1914 - val_loss: 8.7023 - val_acc: 0.2291\n",
      "Epoch 7/100\n",
      "982/982 [==============================] - 353s 360ms/step - loss: 9.8015 - acc: 0.2393 - val_loss: 8.0125 - val_acc: 0.2477\n",
      "Epoch 8/100\n",
      "982/982 [==============================] - 382s 388ms/step - loss: 9.4637 - acc: 0.2475 - val_loss: 7.3396 - val_acc: 0.3096\n",
      "Epoch 9/100\n",
      "982/982 [==============================] - 485s 494ms/step - loss: 8.8494 - acc: 0.2800 - val_loss: 6.6887 - val_acc: 0.3684\n",
      "Epoch 10/100\n",
      "982/982 [==============================] - 386s 393ms/step - loss: 8.0853 - acc: 0.3238 - val_loss: 6.1587 - val_acc: 0.4025\n",
      "Epoch 11/100\n",
      "982/982 [==============================] - 383s 390ms/step - loss: 7.2978 - acc: 0.3442 - val_loss: 5.6500 - val_acc: 0.4644\n",
      "Epoch 12/100\n",
      "982/982 [==============================] - 389s 396ms/step - loss: 6.8963 - acc: 0.3880 - val_loss: 5.1826 - val_acc: 0.4799\n",
      "Epoch 13/100\n",
      "982/982 [==============================] - 439s 447ms/step - loss: 6.5491 - acc: 0.4287 - val_loss: 4.7355 - val_acc: 0.5015\n",
      "Epoch 14/100\n",
      "982/982 [==============================] - 380s 387ms/step - loss: 6.0276 - acc: 0.4715 - val_loss: 4.3283 - val_acc: 0.5325\n",
      "Epoch 15/100\n",
      "982/982 [==============================] - 381s 388ms/step - loss: 5.7917 - acc: 0.4644 - val_loss: 3.9634 - val_acc: 0.5728\n",
      "Epoch 16/100\n",
      "982/982 [==============================] - 384s 391ms/step - loss: 5.2197 - acc: 0.5102 - val_loss: 3.6231 - val_acc: 0.6037\n",
      "Epoch 17/100\n",
      "982/982 [==============================] - 400s 408ms/step - loss: 4.8869 - acc: 0.5234 - val_loss: 3.3228 - val_acc: 0.6347\n",
      "Epoch 18/100\n",
      "982/982 [==============================] - 416s 424ms/step - loss: 4.6531 - acc: 0.5621 - val_loss: 3.0785 - val_acc: 0.6563\n",
      "Epoch 19/100\n",
      "982/982 [==============================] - 383s 390ms/step - loss: 4.2093 - acc: 0.5957 - val_loss: 2.8987 - val_acc: 0.6563\n",
      "Epoch 20/100\n",
      "982/982 [==============================] - 383s 390ms/step - loss: 3.7959 - acc: 0.6232 - val_loss: 2.7602 - val_acc: 0.6811\n",
      "Epoch 21/100\n",
      "982/982 [==============================] - 381s 388ms/step - loss: 3.4101 - acc: 0.6619 - val_loss: 2.6460 - val_acc: 0.7028\n",
      "Epoch 22/100\n",
      "982/982 [==============================] - 387s 394ms/step - loss: 3.4370 - acc: 0.6599 - val_loss: 2.5271 - val_acc: 0.7059\n",
      "Epoch 23/100\n",
      "982/982 [==============================] - 486s 495ms/step - loss: 2.9947 - acc: 0.6823 - val_loss: 2.3959 - val_acc: 0.7121\n",
      "Epoch 24/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "982/982 [==============================] - 337s 343ms/step - loss: 3.0307 - acc: 0.6874 - val_loss: 2.3030 - val_acc: 0.7183\n",
      "Epoch 25/100\n",
      "982/982 [==============================] - 401s 409ms/step - loss: 2.8994 - acc: 0.7189 - val_loss: 2.2441 - val_acc: 0.7245\n",
      "Epoch 26/100\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-317f12fec38c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     93\u001b[0m                                         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mNN_CALLBACKS\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m                                         \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 95\u001b[1;33m                                         verbose=1)\n\u001b[0m\u001b[0;32m     96\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     97\u001b[0m                 \u001b[0mfitting_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0minit_fitting_time\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python\\python36\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1035\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1036\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1037\u001b[1;33m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1038\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1039\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32mc:\\python\\python36\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python\\python36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2664\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2665\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2666\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2667\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2668\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python\\python36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2634\u001b[0m                                 \u001b[0msymbol_vals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2635\u001b[0m                                 session)\n\u001b[1;32m-> 2636\u001b[1;33m         \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2637\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2638\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m   1449\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1450\u001b[0m           return tf_session.TF_SessionRunCallable(\n\u001b[1;32m-> 1451\u001b[1;33m               self._session._session, self._handle, args, status, None)\n\u001b[0m\u001b[0;32m   1452\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1453\u001b[0m           return tf_session.TF_DeprecatedSessionRunCallable(\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "iter_start_time = time.time()\n",
    "\n",
    "print('filtered_races : %s'%L_filtered_races)\n",
    "print()\n",
    "print('building the train and test datasets ...')\n",
    "print()\n",
    "dict_data = build_train_validation_and_test_datasets(L_filtered_races,'label_encoder_iterations','label_encoder_iterations')\n",
    "\n",
    "X_train = dict_data['X_train']\n",
    "X_val = dict_data['X_val']\n",
    "X_test = dict_data['X_test']\n",
    "    \n",
    "y_train = dict_data['y_train']\n",
    "y_val = dict_data['y_val']\n",
    "y_test = dict_data['y_test']\n",
    "            \n",
    "training_len = X_train.shape[0]\n",
    "testing_len = X_test.shape[0]\n",
    "            \n",
    "print('looping over the vgg16 parameters ...')\n",
    "print()\n",
    "for lr_iteration in range(0,20):            \n",
    "    for BATCH_SIZE in L_batch_sizes:\n",
    "        for key in dict_lr_ranges.keys():\n",
    "            lr_interval = dict_lr_ranges[key]\n",
    "            LEARNING_RATE = random.uniform(lr_interval[0], lr_interval[1])\n",
    "            init_training_time = time.time()\n",
    "            print()\n",
    "            print()\n",
    "            print(\"#############################################################################################\")\n",
    "            print(\"training the model with params :\")\n",
    "            print(\"Batch size = %s | Learning rate = %s\"%(BATCH_SIZE,LEARNING_RATE))\n",
    "\n",
    "            for optimizer in ['adam','sgd']:\n",
    "                # Charger VGG-16 pré-entraîné sur ImageNet et sans les couches fully-connected\n",
    "                model = VGG16(weights=\"imagenet\", include_top=False, input_shape=(224, 224, 3))\n",
    "\n",
    "                # choose the layers which are updated by training\n",
    "                for layer in model.layers:\n",
    "                    layer.trainable = False\n",
    "\n",
    "                # Récupérer la sortie de ce réseau\n",
    "                x = model.output\n",
    "                # Ajouter la fonction Flatten à la nouvelle couche fully-connected pour la classification à 2 classes\n",
    "                predictions = Flatten()(x)\n",
    "\n",
    "                # Définir le nouveau modèle\n",
    "                new_model = Model(inputs=model.input, outputs=predictions)\n",
    "\n",
    "                # Ajout d'une couche Fully connected à 4096 neurones ayant une fonction d'activation \"relu\"\n",
    "                x = new_model.output\n",
    "                predictions = Dense(4096, activation='relu')(x)\n",
    "                new_model = Model(inputs=new_model.input, outputs=predictions)\n",
    "\n",
    "                \"\"\"Ajout d'une couche Dropout\"\"\"\n",
    "                x = new_model.output\n",
    "                dropout = Dropout(0.2)(x)\n",
    "                new_model = Model(inputs=new_model.input, outputs=dropout)\n",
    "\n",
    "\n",
    "                # Ajout d'une couche supplémentaire Fully connected à 4096 neurones ayant une fonction d'activation \"relu\"\n",
    "                x = new_model.output\n",
    "                predictions = Dense(4096, activation='relu')(x)\n",
    "                new_model = Model(inputs=new_model.input, outputs=predictions)\n",
    "\n",
    "                \"\"\"Ajout d'une couche Dropout\"\"\"\n",
    "                x = new_model.output\n",
    "                dropout = Dropout(0.2)(x)\n",
    "                new_model = Model(inputs=new_model.input, outputs=dropout)\n",
    "\n",
    "                # Ajout d'une couche Fully connected à 2 neurones ayant une fonction d'activation \"softmax\"\n",
    "                x = new_model.output\n",
    "                predictions = Dense(RACE_NUMBER, activation='softmax')(x)\n",
    "                new_model = Model(inputs=new_model.input, outputs=predictions)\n",
    "\n",
    "\n",
    "                # Compiler le modèle\n",
    "                if optimizer == 'adam':\n",
    "                    nn_optimizer = Adam(lr=LEARNING_RATE, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n",
    "                if optimizer == 'sgd':\n",
    "                    nn_optimizer = SGD(lr=LEARNING_RATE, momentum=0.9)\n",
    "                new_model.compile(loss=\"categorical_crossentropy\", optimizer=nn_optimizer, metrics=[\"accuracy\"])\n",
    "\n",
    "                init_fitting_time = time.time()\n",
    "                print('fitting the model ...')\n",
    "                print('optimizer : %s'%optimizer)\n",
    "                print()\n",
    "                # Entraîner sur les données d'entraînement (X_train, y_train)\n",
    "                history = new_model.fit(X_train,\n",
    "                                        y_train,\n",
    "                                        validation_data=(X_val,y_val),\n",
    "                                        epochs=EPOCHS,\n",
    "                                        callbacks = [NN_CALLBACKS],\n",
    "                                        batch_size=BATCH_SIZE,\n",
    "                                        verbose=1)\n",
    "\n",
    "                fitting_time = int(time.time() - init_fitting_time)\n",
    "                print('fitting_time : %s seconds'%fitting_time)\n",
    "                print()\n",
    "\n",
    "                init_prediction_time = time.time()\n",
    "                print('computing the accuracy on the test data ...')\n",
    "                print()\n",
    "                performances = new_model.evaluate(X_test, y_test, verbose=0)\n",
    "                test_loss = performances[0]\n",
    "                test_accuracy = performances[1]\n",
    "                print(\"test_loss : %s | test_accuracy : %s\"%(test_loss,test_accuracy))\n",
    "                prediction_time = int(time.time() - init_prediction_time)\n",
    "                print('prediction_time : %s seconds'%prediction_time)\n",
    "                print()\n",
    "\n",
    "                dict_evaluation = {\n",
    "                    'training_len':training_len,\n",
    "                    'testing_len':testing_len,\n",
    "                    'n_races':RACE_NUMBER,\n",
    "                    'races':L_filtered_races,\n",
    "                    'batch_size':BATCH_SIZE,\n",
    "                    'learning_rate':LEARNING_RATE,\n",
    "                    'fitting_time':fitting_time,\n",
    "                    'prediction_time':prediction_time,\n",
    "                    'epochs_losses':history.history['loss'],\n",
    "                    'epochs_accuracies':history.history['acc'],\n",
    "                    'epochs_val_losses':history.history['val_loss'],\n",
    "                    'epochs_val_accuracies':history.history['val_acc'],\n",
    "                    'test_loss':test_loss,\n",
    "                    'test_accuracy':test_accuracy,\n",
    "                    'optimizer':optimizer\n",
    "                    }\n",
    "\n",
    "                df_evaluation_final = df_evaluation_final.append(dict_evaluation, ignore_index = True)\n",
    "                df_evaluation_final.to_csv('df_evaluation_final.csv', header=True)\n",
    "\n",
    "                print('CSV written !')\n",
    "                print()\n",
    "                print('training_time : %s seconds'% (time.time() - init_training_time))\n",
    "                print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}