Road_signs_recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "from torchvision import models\n",
    "\n",
    "plt.ion()   # interactive mode\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.5.1\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, criterion, optimizer, scheduler, metric, num_epochs=25):\n",
    "    since = time.time()\n",
    "    \n",
    "    epoch_loss_train = []\n",
    "    epoch_acc_train = []\n",
    "    \n",
    "    epoch_loss_test = []\n",
    "    epoch_acc_test = []\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', metric]:\n",
    "            if phase == 'train':\n",
    "                model.train()  # Set model to training mode\n",
    "            else:\n",
    "                model.eval()   # Set model to evaluate mode\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "                    loss = criterion(outputs, labels)\n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "            if phase == 'train':\n",
    "                scheduler.step()\n",
    "                epoch_loss_train.append(running_loss / dataset_sizes[phase])\n",
    "                epoch_acc_train.append(running_corrects.double() / dataset_sizes[phase])\n",
    "                print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n",
    "                phase, epoch_loss_train[-1], epoch_acc_train[-1]))\n",
    "            else:\n",
    "                epoch_loss_test.append(running_loss / dataset_sizes[phase])\n",
    "                epoch_acc_test.append(running_corrects.double() / dataset_sizes[phase])\n",
    "                print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss_test[-1], epoch_acc_test[-1]))\n",
    "            \n",
    "            # deep copy the model\n",
    "            if phase == metric and epoch_acc_test[-1] > best_acc:\n",
    "                best_acc = epoch_acc_test[-1]\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "\n",
    "            \n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best '+ str(metric) +' accuracy: {:4f}\\n'.format(best_acc))\n",
    "    \n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, (epoch_loss_train, epoch_acc_train), (epoch_loss_test, epoch_acc_test), best_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_acc_plot(acc_loss, who, loss=True):\n",
    "    acc = []\n",
    "    for i in range(len(acc_loss[1])):\n",
    "        acc.append(acc_loss[1][i].cpu().item())\n",
    "    if loss:    \n",
    "        plt.plot(range(len(acc_loss[0])), acc_loss[0], label='loss')\n",
    "    plt.plot(range(len(acc_loss[1])), acc, label='accuracy')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.title(str(who) +': loss and accuracy plot')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_models(models):\n",
    "    for m in models:\n",
    "        acc = []\n",
    "        for i in range(len(m[2][1])):\n",
    "            acc.append(m[2][1][i].cpu().item())\n",
    "        plt.plot(range(len(m[2][1])), acc, label=str(m[0].__class__.__name__))\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.title('Test accuracy')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning rate and momentum tuning (IS NOT NECESSARY TO RUN THE CELL BELOW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using lr= 0.001 and m= 0.5\n",
      "Epoch 0/0\n",
      "----------\n",
      "train Loss: 1.3738 Acc: 0.6115\n",
      "val Loss: 0.2404 Acc: 0.9467\n",
      "Training complete in 0m 50s\n",
      "Best val accuracy: 0.946746\n",
      "\n",
      "Using lr= 0.001 and m= 0.7\n",
      "Epoch 0/0\n",
      "----------\n",
      "train Loss: 0.6008 Acc: 0.8461\n",
      "val Loss: 0.0562 Acc: 0.9882\n",
      "Training complete in 0m 51s\n",
      "Best val accuracy: 0.988166\n",
      "\n",
      "Using lr= 0.005 and m= 0\n",
      "Epoch 0/0\n",
      "----------\n",
      "train Loss: 0.4625 Acc: 0.8700\n",
      "val Loss: 0.0373 Acc: 0.9921\n",
      "Training complete in 0m 51s\n",
      "Best val accuracy: 0.992110\n",
      "\n",
      "Using lr= 0.005 and m= 1\n",
      "Epoch 0/0\n",
      "----------\n",
      "train Loss: 0.4124 Acc: 0.8821\n",
      "val Loss: 0.0350 Acc: 0.9875\n",
      "Training complete in 0m 51s\n",
      "Best val accuracy: 0.987508\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(tensor(0.9467, device='cuda:0', dtype=torch.float64), 0.001, 0.5),\n",
       " (tensor(0.9882, device='cuda:0', dtype=torch.float64), 0.001, 0.7),\n",
       " (tensor(0.9921, device='cuda:0', dtype=torch.float64), 0.005, 0.5),\n",
       " (tensor(0.9875, device='cuda:0', dtype=torch.float64), 0.005, 0.7)]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.Resize(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "\n",
    "data_dir = 'traffic'\n",
    "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),\n",
    "                                          data_transforms[x])\n",
    "                  for x in ['train', 'val']}\n",
    "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4,\n",
    "                                             shuffle=True, num_workers=4)\n",
    "              for x in ['train', 'val']}\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n",
    "class_names = image_datasets['train'].classes\n",
    "\n",
    "learning_rate = [0.001, 0.005, 0.1]\n",
    "momentum = [0.3, 0.5, 0.7, 0.9]\n",
    "\n",
    "model = torchvision.models.resnet18(pretrained=True)\n",
    "num_ftrs = model.fc.in_features\n",
    "model.fc = nn.Linear(num_ftrs, 21)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "accs = []\n",
    "\n",
    "for i in range(2):\n",
    "    print('Using lr: ' + str(learning_rate[0]) + ' and m: ' + str(momentum[i]))\n",
    "    if i == 0:\n",
    "        optimizer = optim.SGD(model.parameters(), lr=learning_rate[0], momentum=momentum[i])\n",
    "        exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n",
    "        model = train_model(model, criterion, optimizer, exp_lr_scheduler, metric='val', num_epochs=25)\n",
    "        accs.append((model[3], learning_rate[0], momentum[i]))\n",
    "    else:\n",
    "        optimizer = optim.SGD(model[0].parameters(), lr=learning_rate[0],momentum=momentum[i])\n",
    "        exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n",
    "        model = train_model(model[0], criterion, optimizer, exp_lr_scheduler, metric='val', num_epochs=25)\n",
    "        accs.append((model[3], learning_rate[0], momentum[i]))\n",
    "\n",
    "for i in range(1, len(learning_rate)):\n",
    "    for m in range(len(momentum)):\n",
    "        print('Using lr: ' + str(learning_rate[i]) + ' and m: ' + str(momentum[m]))\n",
    "        optimizer = optim.SGD(model[0].parameters(), lr=learning_rate[i],momentum=momentum[m])\n",
    "        exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n",
    "        model = train_model(model[0], criterion, optimizer, exp_lr_scheduler, metric='val', num_epochs=25)\n",
    "        accs.append((model[3], learning_rate[i], momentum[m]))\n",
    "\n",
    "best_params = []\n",
    "for i in range(len(accs)):\n",
    "    best_params.append(accs[i][0].cpu().item())\n",
    "# best val\n",
    "learning_rate = accs[np.argmax(best_params)][1] # 1 is for learning rate\n",
    "momentum = accs[np.argmax(best_params)][2] # 2 is for momentum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tuning step size and gamma (IS NOT NECESSARY TO RUN THE CELL BELOW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "step_size = [1, 5, 7, 10]\n",
    "gamma = [0.1, 0.5, 1]\n",
    "\n",
    "model = torchvision.models.resnet18(pretrained=True)\n",
    "num_ftrs = model.fc.in_features\n",
    "model.fc = nn.Linear(num_ftrs, 21)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "accs = []\n",
    "\n",
    "for i in range(2):\n",
    "    print('Using step_size: ' + str(step_size[0]) + ' and gamma: ' + str(gamma[i]))\n",
    "    if i == 0:\n",
    "        optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=momentum)\n",
    "        exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size[0], gamma=gamma[i])\n",
    "        model = train_model(model, criterion, optimizer, exp_lr_scheduler, metric='val', num_epochs=25)\n",
    "        accs.append((model[3], step_size[0], gamma[i]))\n",
    "    else:\n",
    "        optimizer = optim.SGD(model[0].parameters(), lr=learning_rate, momentum=momentum)\n",
    "        exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size[0], gamma=gamma[i])\n",
    "        model = train_model(model[0], criterion, optimizer, exp_lr_scheduler, metric='val', num_epochs=25)\n",
    "        accs.append((model[3], step_size[0], gamma[i]))\n",
    "\n",
    "for i in range(1, len(step_size)):\n",
    "    for g in range(len(gamma)):\n",
    "        print('Using step_size: ' + str(step_size[i]) + ' and gamma: ' + str(gamma[g]))\n",
    "        optimizer = optim.SGD(model[0].parameters(), lr=learning_rate ,momentum=momentum)\n",
    "        exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size[i], gamma=gamma[g])\n",
    "        model = train_model(model[0], criterion, optimizer, exp_lr_scheduler, metric='val', num_epochs=25)\n",
    "        accs.append((model[3], step_size[i], gamma[g]))\n",
    "        \n",
    "best_params = []\n",
    "for i in range(len(accs)):\n",
    "    best_params.append(accs[i][0].cpu().item())\n",
    "# best val\n",
    "step_size = accs[np.argmax(best_params)][1] # 1 is for step_size\n",
    "gamma = accs[np.argmax(best_params)][2] # 2 is for gamma"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From now on, you should run the code below to train and test the model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Just normalization for validation\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'test': transforms.Compose([\n",
    "        transforms.Resize(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "\n",
    "data_dir = 'traffic'\n",
    "output_feature = 21\n",
    "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),\n",
    "                                          data_transforms[x])\n",
    "                  for x in ['train', 'test']}\n",
    "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4,\n",
    "                                             shuffle=True, num_workers=4)\n",
    "              for x in ['train', 'test']}\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'test']}\n",
    "class_names = image_datasets['train'].classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# best hyperparameters values found\n",
    "learning_rate = 0.005\n",
    "momentum = 0.5\n",
    "step_size = 10\n",
    "gamma = 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/24\n",
      "----------\n",
      "train Loss: 0.8093 Acc: 0.7718\n",
      "test Loss: 0.2930 Acc: 0.9531\n",
      "Epoch 1/24\n",
      "----------\n",
      "train Loss: 0.3752 Acc: 0.8952\n",
      "test Loss: 0.2328 Acc: 0.9531\n",
      "Epoch 2/24\n",
      "----------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-38-09836b74575c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mexp_lr_scheduler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr_scheduler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStepLR\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstep_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstep_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgamma\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgamma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexp_lr_scheduler\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_epochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m25\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-33-ad9cc427e04d>\u001b[0m in \u001b[0;36mtrain_model\u001b[1;34m(model, criterion, optimizer, scheduler, metric, num_epochs)\u001b[0m\n\u001b[0;32m     45\u001b[0m                         \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m                 \u001b[0mrunning_loss\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     48\u001b[0m                 \u001b[0mrunning_corrects\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpreds\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mphase\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "num_ftrs = model.fc.in_features\n",
    "model.fc = nn.Linear(num_ftrs, output_feature)\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=momentum)\n",
    "\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
    "\n",
    "model = train_model(model, criterion, optimizer, exp_lr_scheduler, metric='test', num_epochs=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy and loss on training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_acc_plot(model[1], who='training') # model[1] is for training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy and loss on testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_acc_plot(model[2], who='testing') # model[2] is for testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2 = torchvision.models.alexnet(pretrained=True)\n",
    "model2.classifier[-1] = nn.Linear(in_features=4096, out_features=output_feature)\n",
    "\n",
    "model2 = model2.to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model2.parameters(), lr=learning_rate, momentum=momentum)\n",
    "\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
    "\n",
    "model2 = train_model(model2, criterion, optimizer, exp_lr_scheduler, metric='test', num_epochs=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy and loss on training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_acc_plot(model2[1], who='training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy and loss on testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_acc_plot(model2[2], who='testing')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VGG16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model3 = torchvision.models.vgg16(pretrained=True)\n",
    "model3.classifier[-1] = nn.Linear(in_features=4096, out_features=output_feature)\n",
    "\n",
    "model3 = model3.to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model3.parameters(), lr=learning_rate, momentum=momentum)\n",
    "\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
    "\n",
    "model3 = train_model(model3, criterion, optimizer, exp_lr_scheduler, metric='test', num_epochs=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy and loss on training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_acc_plot(model3[1], who='training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy and loss on testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_acc_plot(model3[2], who='testing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_models([model, model2, model3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Documentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PART I"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**IMPORTANT NOTES**: for a correct execution of the program you should have pytorch installed with version 1.4.0 along all its libraries I am using in the code.\n",
    "\n",
    "The cells where on the heading is written \"IS NOT NECESSARY TO RUN THE CELL BELOW\" means that the underneath cell is not necessary to be ran because it does the tuning of some parameters whose best values have already been found (because I ran that code on my computer)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Data preparation**: the given data was structured in train and test set. Moreover these data was splitted in two folders (Belgium and Germany). I decided to merge the data of these two folders both for the training and test set. Moreover, to do hyperparameter tuning, I splitted the training set into two sets: training and validation set.\n",
    "So now the data has this structure:\n",
    "\n",
    "    traffic/\n",
    "        - train/ (training set)\n",
    "        - val/ (validation set)\n",
    "        - test/ (test set)\n",
    "\n",
    "\n",
    "The function that trains the model is the functon `train_model`. At the beginning my first attempt has been to use *Transfer Learning* using a pretrained ImageNet model and apply fine tuning on it. However, at test time, this turns out to be not efficient at all since the accuracy of the model peaked at roughly 80%. <br>\n",
    "Therefore I decided to still keep the model's structure of the ImageNet models I tried before, but without fine tuning them. So updating all the model's parameters again when training it with my data (of course I had to change the output layer of the network to my specific case). This boosted the accuracy of the model to ~90%. <br>\n",
    "\n",
    "In order to still raise the accuracy, I applied *data augmentation* and *hyperparameter tunig*. For data augmentation has been important to look first at the images of the road signs  since some operations like vertical flipping should have not been applied since this could have causes a road sign to look similar to another sign after such transformation.\n",
    "The tuned hyperparameters alongside their values are the following:\n",
    "\n",
    "```python\n",
    "    # best hyperparameters values found\n",
    "    learning_rate = 0.005\n",
    "    momentum = 0.5\n",
    "    step_size = 10\n",
    "    gamma = 0.5\n",
    "```\n",
    "These are used in the optimization function Stochastic Gradient Descent(SGD) and Step Learning Rate(StepLR).\n",
    "\n",
    "Done that, I tested three different model architecture of ImageNet, in particular **ResNet18**, **AlexNet** and **VGG16**. The best performance has been reached by ResNet18 (as it can be seen from the chart that the function `compare_models` outputs -- note that you have to train the three models and then run the last cell `compare_models([model, model2, model3])` to see the chart) reaching ~98-99% of accuracy on the test set after 25 epochs and a batch size of 4.\n",
    "\n",
    "### Deployment of the model on edge device\n",
    "\n",
    "In order to build a model that is suitable for deployment on edge devices the first thing that comes to my mind was to choose a network architecture that is much lighter (in terms of number of layers) than the architectures I used here. Since this is a custom problem, one could think of building a custom CNN specific for this problem and then test its performace, and actually this can be easily done. The main problem, however, concerns with the edge device characteristics. Is is very likely that the resources like RAM, CPU, GPU and power are very limited, therefore I would look for frameworks that have been specifically designed for this like Tensorflow Lite or similar."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PART II"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To deal with this part, I trained a model based on just the data labeled with 12 and 13 to see how difficult was to classify them (namely left and right curve) using the same process as before (namely creating a validation set and finding the best hyperparameters for the model). Here an important thing to notice is that, when I do data augmentation, I should not apply the horizontal nor vertical flipping of the data. In fact, doing so, the performance drops dramatically.\n",
    "\n",
    "NOTE: To train the model based on just these two labels, it is sufficient to change the line of code `data_dir = 'traffic'` under the heading \"Train and test\" to `data_dir = 'left_right'` and then change the line immediately below `output_feature = 21` to `output_feature=2` and run everything again.\n",
    "\n",
    "The trained model peaked a very high accuracy on the test set, sometimes also 100% and an accuracy of ~90% on the training set. This makes me think a little bit. At the beginning I thought this was realted to an underfitting but therefore I decided to look at the data I was training and testing on and I noticed that the test data where few (a dozens) and they were much cleaner than the training data. So I concluded that the model has been trained enough to correctly classify the images on the test set. Contrarily, some images in the training set were blurry and skewed so it is normal for the training set having a lower accuracy since it is much difficult for the model to correctly classify that type of road signs.\n",
    "However, I am aware that a 100% of accuracy on the test set is quite unrealistic, this is probably due to the not-so-good split of the data.\n",
    "\n",
    "Having done these experiments on just these two labels I decided to not doing further modification to my orginal model since on these two labels I had 100% of accuracy on the test set. Of course I paid attention to not apply wrong data augmentation transformations."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}