Knowledge_Distillation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VJvQzH1sNQ6A"
   },
   "source": [
    "## **Load Libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "RPo7Mkm94TCp"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import copy\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader, random_split, Subset\n",
    "from torchvision import datasets, transforms, models\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0isEDkh2Na94"
   },
   "source": [
    "## **Parameter Initialization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "7TiSwBjkSnN6"
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "learning_rate = 0.001\n",
    "epochs = 10\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JGjfMYEyNhBb"
   },
   "source": [
    "## **Data Loading and Preparation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "CPpzsA_iPOND"
   },
   "outputs": [],
   "source": [
    "\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.5,0.5,0.5], std=[0.5,0.5,0.5])\n",
    "])\n",
    "\n",
    "def load_dataset():\n",
    "  train_data = datasets.CIFAR10(root='/content/data', train=True, download=True, transform=transform)\n",
    "  train_size = int(0.8 * len(train_data))\n",
    "  val_size = len(train_data) - train_size\n",
    "  train_data, val_data = random_split(train_data, [train_size, val_size])\n",
    "  test_data = datasets.CIFAR10(root='/content/data', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "  train_subset_indices, test_subset_indices = list(range(batch_size*4 * 24)), list(range(batch_size*4 * 8))\n",
    "  train_subset = Subset(train_data, train_subset_indices)\n",
    "  val_data = Subset(val_data, test_subset_indices)\n",
    "  test_data = Subset(test_data, test_subset_indices)\n",
    "\n",
    "  return train_subset, val_data, test_data\n",
    "\n",
    "def prepare_data(train_data, val_data, test_data=None):\n",
    "  train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
    "  val_loader = DataLoader(val_data, batch_size=batch_size, shuffle=False)\n",
    "  if test_data:\n",
    "    test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)\n",
    "    return train_loader, val_loader, test_loader\n",
    "\n",
    "  return train_loader, val_loader\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sZ62l0ESNn1P"
   },
   "source": [
    "## **Model Loading**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "z0uQnbAcOQ9u"
   },
   "outputs": [],
   "source": [
    "def load_model(pretrained=True, width_mult=1.0):\n",
    "  model = models.mobilenet_v2(pretrained=pretrained, width_mult=width_mult)\n",
    "  model.classifier[1] = nn.Linear(model.last_channel, 10)\n",
    "  return model.to(device)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i6YXT27ON0mc"
   },
   "source": [
    "## **Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "F6lyj4VzZOYj"
   },
   "outputs": [],
   "source": [
    "def evaluate_model(model, test_loader):\n",
    "    model.eval()\n",
    "    all_labels = []\n",
    "    all_predictions = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for images, labels in tqdm(test_loader):\n",
    "            images, labels = images.to(device), labels\n",
    "            outputs = model(images)\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "\n",
    "            all_labels.extend(labels.numpy())\n",
    "            all_predictions.extend(predicted.cpu().numpy())\n",
    "\n",
    "    # Confusion matrix\n",
    "    cm = confusion_matrix(all_labels, all_predictions)\n",
    "    print(\"Confusion Matrix:\")\n",
    "    print(cm)\n",
    "\n",
    "    # Classification report (Precision, Recall, F1-score)\n",
    "    report = classification_report(all_labels, all_predictions)\n",
    "    print(\"Classification Report:\")\n",
    "    print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Y3x-K-EKN53J"
   },
   "source": [
    "## **Fine-tuning**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "Cm883Bsq3KXX"
   },
   "outputs": [],
   "source": [
    "def train_model(model, train_loader, val_loader, epochs, gradient_acc_steps = 256):\n",
    "  criterion = nn.CrossEntropyLoss()\n",
    "  optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "  best_model = copy.deepcopy(model).to(device)\n",
    "  best_accuracy = 0.0\n",
    "\n",
    "  for epoch in range(epochs):\n",
    "    print(f\"*** Epoch: {epoch+1} ***\")\n",
    "    model.train()\n",
    "    running_loss = 0.0\n",
    "    n_steps = 0\n",
    "    bar = tqdm(train_loader)\n",
    "    for images, labels in bar:\n",
    "\n",
    "      images, labels = images.to(device), labels.to(device)\n",
    "      outputs = model(images)\n",
    "\n",
    "      unit_loss = criterion(outputs, labels)*len(images)/gradient_acc_steps\n",
    "      unit_loss.backward()\n",
    "      running_loss += unit_loss.detach().item()\n",
    "\n",
    "      n_steps += len(images)\n",
    "      if n_steps % gradient_acc_steps == 0:\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        optimizer.step()\n",
    "\n",
    "        bar.set_description(f\"Loss: {running_loss:.5f}\")\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        running_loss = 0.0\n",
    "\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "      correct = 0\n",
    "      total = 0\n",
    "      for images, labels in val_loader:\n",
    "        images, labels = images.to(device), labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "      val_accuracy = 100 * correct / total\n",
    "\n",
    "      print(f\"Validation Accuracy: {val_accuracy:.2f}%\")\n",
    "\n",
    "      if val_accuracy > best_accuracy:\n",
    "        best_accuracy = val_accuracy\n",
    "        best_model = copy.deepcopy(model).to(device)\n",
    "\n",
    "  return best_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lTM4sqssOG5H"
   },
   "source": [
    "## **Model Distillation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "cUV-VE-CCc1C"
   },
   "outputs": [],
   "source": [
    "def distillation_loss(teacher_outputs, student_outputs, labels, alpha, temperature):\n",
    "  teacher_probs = nn.functional.softmax(teacher_outputs / temperature, dim=1)\n",
    "  student_probs = nn.functional.log_softmax(student_outputs / temperature, dim=1)\n",
    "  kd_loss = nn.functional.kl_div(student_probs, teacher_probs, reduction='batchmean')*(temperature**2)\n",
    "  ce_loss = nn.functional.cross_entropy(student_outputs, labels)\n",
    "  return alpha * kd_loss + (1 - alpha) * ce_loss\n",
    "\n",
    "\n",
    "def model_distillation(teacher_model, student_model, train_loader, val_loader,\n",
    "                    epochs, alpha=0.5, temperature=3, gradient_acc_steps=256):\n",
    "\n",
    "  optimizer = torch.optim.Adam(student_model.parameters(), lr=learning_rate)\n",
    "\n",
    "\n",
    "  best_model = copy.deepcopy(student_model).to(device)\n",
    "  best_accuracy = 0.0\n",
    "\n",
    "  teacher_model.eval()\n",
    "\n",
    "  for epoch in range(epochs):\n",
    "    print(f\"*** Epoch: {epoch+1} ***\")\n",
    "    student_model.train()\n",
    "    running_loss = 0.0\n",
    "\n",
    "    n_steps = 0\n",
    "    bar = tqdm(train_loader)\n",
    "    for images, labels in bar:\n",
    "      images, labels = images.to(device), labels.to(device)\n",
    "      with torch.no_grad():\n",
    "        teacher_outputs = teacher_model(images)\n",
    "      student_outputs = student_model(images)\n",
    "\n",
    "      unit_loss = distillation_loss(\n",
    "          teacher_outputs, student_outputs, labels, alpha, temperature\n",
    "      )*len(images)/gradient_acc_steps\n",
    "      unit_loss.backward()\n",
    "      running_loss += unit_loss.detach().item()\n",
    "\n",
    "      n_steps += len(images)\n",
    "      if n_steps % gradient_acc_steps == 0:\n",
    "        nn.utils.clip_grad_norm_(student_model.parameters(), 1.0)\n",
    "        optimizer.step()\n",
    "\n",
    "        bar.set_description(f\"Loss: {running_loss:.5f}\")\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        running_loss = 0.0\n",
    "\n",
    "    student_model.eval()\n",
    "    with torch.no_grad():\n",
    "      teacher_correct = 0\n",
    "      student_correct = 0\n",
    "      total = 0\n",
    "      for images, labels in val_loader:\n",
    "        images, labels = images.to(device), labels.to(device)\n",
    "        teacher_outputs = teacher_model(images)\n",
    "        student_outputs = student_model(images)\n",
    "        _, teacher_predicted = torch.max(teacher_outputs.data, 1)\n",
    "        _, student_predicted = torch.max(student_outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        teacher_correct += (teacher_predicted == labels).sum().item()\n",
    "        student_correct += (student_predicted == labels).sum().item()\n",
    "\n",
    "\n",
    "      val_accuracy = 100 * student_correct / total\n",
    "\n",
    "      if val_accuracy > best_accuracy:\n",
    "        best_accuracy = val_accuracy\n",
    "        best_model = copy.deepcopy(student_model).to(device)\n",
    "\n",
    "      print(f\"Validation Accuracy:- Teacher: {100*teacher_correct/total:.2f}%; Student: {val_accuracy:.2f}%\")\n",
    "\n",
    "  return best_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gR2OkqqMOMMj"
   },
   "source": [
    "## **main**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "collapsed": true,
    "id": "yN70hz0kUoeZ",
    "outputId": "9c8cbfc1-4fbc-4e6f-a7d0-68fc32444172"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to /content/data/cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 170M/170M [00:03<00:00, 48.7MB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /content/data/cifar-10-python.tar.gz to /content/data\n",
      "Files already downloaded and verified\n",
      "Train, validation, and test data size:\n",
      "1536 512 512\n",
      "Train, validation, and test data loader size:\n",
      "96 32 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/mobilenet_v2-b0353104.pth\" to /root/.cache/torch/hub/checkpoints/mobilenet_v2-b0353104.pth\n",
      "100%|██████████| 13.6M/13.6M [00:00<00:00, 62.1MB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial teacher Model Performance:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:04<00:00,  7.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      "[[ 8 32  0  4  8  0  0  0  0  0]\n",
      " [18 15  6  4 14  0  0  0  0  0]\n",
      " [ 8 31  1  2 13  0  0  1  0  0]\n",
      " [ 9 26  5  2 13  0  0  0  0  0]\n",
      " [ 7 20 10  1 10  0  0  0  0  0]\n",
      " [ 4 14  8  0 17  0  0  0  0  0]\n",
      " [ 7 24  1  1  7  0  0  0  0  0]\n",
      " [ 9 19  8  2 17  0  0  0  0  0]\n",
      " [ 3 36  2  4  5  0  0  1  0  0]\n",
      " [17 19 10  8  1  0  0  0  0  0]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.09      0.15      0.11        52\n",
      "           1       0.06      0.26      0.10        57\n",
      "           2       0.02      0.02      0.02        56\n",
      "           3       0.07      0.04      0.05        55\n",
      "           4       0.10      0.21      0.13        48\n",
      "           5       0.00      0.00      0.00        43\n",
      "           6       0.00      0.00      0.00        40\n",
      "           7       0.00      0.00      0.00        55\n",
      "           8       0.00      0.00      0.00        51\n",
      "           9       0.00      0.00      0.00        55\n",
      "\n",
      "    accuracy                           0.07       512\n",
      "   macro avg       0.03      0.07      0.04       512\n",
      "weighted avg       0.03      0.07      0.04       512\n",
      "\n",
      "\n",
      "Initial Student Model Perfomance:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:01<00:00, 26.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      "[[ 0  0  0  0  0  0  0 52  0  0]\n",
      " [ 0  0  0  0  0  0  0 57  0  0]\n",
      " [ 0  0  0  0  0  0  0 56  0  0]\n",
      " [ 0  0  0  0  0  0  0 55  0  0]\n",
      " [ 0  0  0  0  0  0  0 48  0  0]\n",
      " [ 0  0  0  0  0  0  0 43  0  0]\n",
      " [ 0  0  0  0  0  0  0 40  0  0]\n",
      " [ 0  0  0  0  0  0  0 55  0  0]\n",
      " [ 0  0  0  0  0  0  0 51  0  0]\n",
      " [ 0  0  0  0  0  0  0 55  0  0]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00        52\n",
      "           1       0.00      0.00      0.00        57\n",
      "           2       0.00      0.00      0.00        56\n",
      "           3       0.00      0.00      0.00        55\n",
      "           4       0.00      0.00      0.00        48\n",
      "           5       0.00      0.00      0.00        43\n",
      "           6       0.00      0.00      0.00        40\n",
      "           7       0.11      1.00      0.19        55\n",
      "           8       0.00      0.00      0.00        51\n",
      "           9       0.00      0.00      0.00        55\n",
      "\n",
      "    accuracy                           0.11       512\n",
      "   macro avg       0.01      0.10      0.02       512\n",
      "weighted avg       0.01      0.11      0.02       512\n",
      "\n",
      "\n",
      "Training Teacher Model:\n",
      "*** Epoch: 1 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.05923: 100%|██████████| 96/96 [00:08<00:00, 11.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 71.29%\n",
      "*** Epoch: 2 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.48875: 100%|██████████| 96/96 [00:07<00:00, 12.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 80.08%\n",
      "*** Epoch: 3 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.30681: 100%|██████████| 96/96 [00:07<00:00, 12.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 78.52%\n",
      "*** Epoch: 4 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.20244: 100%|██████████| 96/96 [00:07<00:00, 12.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 80.47%\n",
      "*** Epoch: 5 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.14286: 100%|██████████| 96/96 [00:07<00:00, 12.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 76.95%\n",
      "*** Epoch: 6 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.14228: 100%|██████████| 96/96 [00:07<00:00, 12.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 79.30%\n",
      "*** Epoch: 7 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.09280: 100%|██████████| 96/96 [00:08<00:00, 10.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 79.69%\n",
      "*** Epoch: 8 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.10597: 100%|██████████| 96/96 [00:07<00:00, 12.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 80.47%\n",
      "*** Epoch: 9 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.10677: 100%|██████████| 96/96 [00:08<00:00, 11.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 80.08%\n",
      "*** Epoch: 10 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.08899: 100%|██████████| 96/96 [00:07<00:00, 12.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 79.88%\n",
      "\n",
      "Teacher Model Performance after training:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:01<00:00, 20.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      "[[46  1  2  0  0  0  0  0  3  0]\n",
      " [ 1 50  0  0  0  0  1  0  1  4]\n",
      " [ 3  0 43  2  1  3  1  3  0  0]\n",
      " [ 1  0  3 40  2  8  1  0  0  0]\n",
      " [ 2  0  3  1 35  2  1  4  0  0]\n",
      " [ 1  0  1  7  3 29  0  2  0  0]\n",
      " [ 1  0  2  1  2  0 34  0  0  0]\n",
      " [ 0  0  2  0  2  3  0 48  0  0]\n",
      " [ 7  2  0  1  0  0  0  1 40  0]\n",
      " [ 4  1  0  1  0  0  1  0  1 47]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.88      0.78        52\n",
      "           1       0.93      0.88      0.90        57\n",
      "           2       0.77      0.77      0.77        56\n",
      "           3       0.75      0.73      0.74        55\n",
      "           4       0.78      0.73      0.75        48\n",
      "           5       0.64      0.67      0.66        43\n",
      "           6       0.87      0.85      0.86        40\n",
      "           7       0.83      0.87      0.85        55\n",
      "           8       0.89      0.78      0.83        51\n",
      "           9       0.92      0.85      0.89        55\n",
      "\n",
      "    accuracy                           0.80       512\n",
      "   macro avg       0.81      0.80      0.80       512\n",
      "weighted avg       0.81      0.80      0.81       512\n",
      "\n",
      "\n",
      "Training Student Model:\n",
      "*** Epoch: 1 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.20450: 100%|██████████| 96/96 [00:05<00:00, 16.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 15.62%\n",
      "*** Epoch: 2 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.10749: 100%|██████████| 96/96 [00:05<00:00, 18.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 17.97%\n",
      "*** Epoch: 3 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.07493: 100%|██████████| 96/96 [00:04<00:00, 19.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 20.90%\n",
      "*** Epoch: 4 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.03340: 100%|██████████| 96/96 [00:05<00:00, 19.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 22.46%\n",
      "*** Epoch: 5 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.97457: 100%|██████████| 96/96 [00:05<00:00, 19.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 22.46%\n",
      "*** Epoch: 6 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.83396: 100%|██████████| 96/96 [00:05<00:00, 18.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 25.39%\n",
      "*** Epoch: 7 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.75444: 100%|██████████| 96/96 [00:04<00:00, 19.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 28.91%\n",
      "*** Epoch: 8 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.71557: 100%|██████████| 96/96 [00:04<00:00, 19.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 29.30%\n",
      "*** Epoch: 9 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.71089: 100%|██████████| 96/96 [00:05<00:00, 18.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 29.49%\n",
      "*** Epoch: 10 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.77115: 100%|██████████| 96/96 [00:04<00:00, 19.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 32.42%\n",
      "*** Epoch: 11 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.76063: 100%|██████████| 96/96 [00:05<00:00, 18.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 35.35%\n",
      "*** Epoch: 12 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.57590: 100%|██████████| 96/96 [00:04<00:00, 19.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 33.59%\n",
      "*** Epoch: 13 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.50320: 100%|██████████| 96/96 [00:05<00:00, 17.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 35.16%\n",
      "*** Epoch: 14 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.51017: 100%|██████████| 96/96 [00:04<00:00, 20.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 35.35%\n",
      "*** Epoch: 15 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.64003: 100%|██████████| 96/96 [00:05<00:00, 17.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.28%\n",
      "*** Epoch: 16 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.45530: 100%|██████████| 96/96 [00:04<00:00, 20.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.65%\n",
      "*** Epoch: 17 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.44463: 100%|██████████| 96/96 [00:05<00:00, 17.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 35.94%\n",
      "*** Epoch: 18 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.42594: 100%|██████████| 96/96 [00:04<00:00, 19.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 35.94%\n",
      "*** Epoch: 19 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.42978: 100%|██████████| 96/96 [00:05<00:00, 17.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.09%\n",
      "*** Epoch: 20 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.41897: 100%|██████████| 96/96 [00:04<00:00, 20.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 36.91%\n",
      "*** Epoch: 21 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.45703: 100%|██████████| 96/96 [00:05<00:00, 17.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.26%\n",
      "*** Epoch: 22 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.47177: 100%|██████████| 96/96 [00:04<00:00, 19.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.06%\n",
      "*** Epoch: 23 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.28698: 100%|██████████| 96/96 [00:05<00:00, 16.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 36.72%\n",
      "*** Epoch: 24 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.25011: 100%|██████████| 96/96 [00:04<00:00, 19.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.65%\n",
      "*** Epoch: 25 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.29690: 100%|██████████| 96/96 [00:05<00:00, 17.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.28%\n",
      "*** Epoch: 26 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.32983: 100%|██████████| 96/96 [00:05<00:00, 17.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.65%\n",
      "*** Epoch: 27 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.23477: 100%|██████████| 96/96 [00:05<00:00, 16.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.09%\n",
      "*** Epoch: 28 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.19810: 100%|██████████| 96/96 [00:04<00:00, 19.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.28%\n",
      "*** Epoch: 29 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.19443: 100%|██████████| 96/96 [00:05<00:00, 17.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 36.72%\n",
      "*** Epoch: 30 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.04127: 100%|██████████| 96/96 [00:04<00:00, 19.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.09%\n",
      "*** Epoch: 31 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.09074: 100%|██████████| 96/96 [00:05<00:00, 17.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 36.72%\n",
      "*** Epoch: 32 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.09229: 100%|██████████| 96/96 [00:04<00:00, 19.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.67%\n",
      "*** Epoch: 33 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.09169: 100%|██████████| 96/96 [00:05<00:00, 18.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.09%\n",
      "*** Epoch: 34 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.96861: 100%|██████████| 96/96 [00:04<00:00, 19.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.87%\n",
      "*** Epoch: 35 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.95252: 100%|██████████| 96/96 [00:04<00:00, 19.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 37.50%\n",
      "*** Epoch: 36 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.94036: 100%|██████████| 96/96 [00:05<00:00, 18.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.09%\n",
      "*** Epoch: 37 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.91202: 100%|██████████| 96/96 [00:04<00:00, 20.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.26%\n",
      "*** Epoch: 38 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.87713: 100%|██████████| 96/96 [00:05<00:00, 17.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.67%\n",
      "*** Epoch: 39 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.87994: 100%|██████████| 96/96 [00:04<00:00, 19.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 37.11%\n",
      "*** Epoch: 40 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.74023: 100%|██████████| 96/96 [00:05<00:00, 17.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.67%\n",
      "*** Epoch: 41 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.78921: 100%|██████████| 96/96 [00:04<00:00, 19.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.67%\n",
      "*** Epoch: 42 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.69435: 100%|██████████| 96/96 [00:05<00:00, 17.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 37.11%\n",
      "*** Epoch: 43 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.63540: 100%|██████████| 96/96 [00:04<00:00, 20.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.84%\n",
      "*** Epoch: 44 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.64176: 100%|██████████| 96/96 [00:05<00:00, 17.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.26%\n",
      "*** Epoch: 45 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.66434: 100%|██████████| 96/96 [00:04<00:00, 19.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.65%\n",
      "*** Epoch: 46 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.56405: 100%|██████████| 96/96 [00:05<00:00, 17.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 37.70%\n",
      "*** Epoch: 47 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.53949: 100%|██████████| 96/96 [00:04<00:00, 19.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.45%\n",
      "*** Epoch: 48 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.62836: 100%|██████████| 96/96 [00:05<00:00, 16.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 39.06%\n",
      "*** Epoch: 49 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.52210: 100%|██████████| 96/96 [00:04<00:00, 20.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 38.67%\n",
      "*** Epoch: 50 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 0.53158: 100%|██████████| 96/96 [00:05<00:00, 17.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 37.30%\n",
      "\n",
      "Student Model Performance after training:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:00<00:00, 34.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      "[[21  3  7  5  2  0  1  0  8  5]\n",
      " [ 2 27  3  1  0  3  1  0  7 13]\n",
      " [ 5  0 22  8  4 10  4  2  0  1]\n",
      " [ 1  0  9 21  4  9  7  1  1  2]\n",
      " [ 5  2  7 10  8  4  8  2  2  0]\n",
      " [ 3  1  4 11  2 17  2  2  1  0]\n",
      " [ 1  0  7  6  1  4 15  3  3  0]\n",
      " [ 1  0  4  3  6 10  4 20  1  6]\n",
      " [11  4  2  2  1  3  0  0 28  0]\n",
      " [ 2 13  0  3  2  1  2  0  7 25]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.40      0.40      0.40        52\n",
      "           1       0.54      0.47      0.50        57\n",
      "           2       0.34      0.39      0.36        56\n",
      "           3       0.30      0.38      0.34        55\n",
      "           4       0.27      0.17      0.21        48\n",
      "           5       0.28      0.40      0.33        43\n",
      "           6       0.34      0.38      0.36        40\n",
      "           7       0.67      0.36      0.47        55\n",
      "           8       0.48      0.55      0.51        51\n",
      "           9       0.48      0.45      0.47        55\n",
      "\n",
      "    accuracy                           0.40       512\n",
      "   macro avg       0.41      0.40      0.39       512\n",
      "weighted avg       0.42      0.40      0.40       512\n",
      "\n",
      "\n",
      "Distillation:\n",
      "*** Epoch: 1 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 5.66864: 100%|██████████| 96/96 [00:06<00:00, 15.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 15.43%\n",
      "*** Epoch: 2 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 5.37534: 100%|██████████| 96/96 [00:06<00:00, 15.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 22.07%\n",
      "*** Epoch: 3 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 5.27867: 100%|██████████| 96/96 [00:06<00:00, 15.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 22.27%\n",
      "*** Epoch: 4 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.85715: 100%|██████████| 96/96 [00:05<00:00, 17.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 24.61%\n",
      "*** Epoch: 5 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.74464: 100%|██████████| 96/96 [00:06<00:00, 15.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 24.61%\n",
      "*** Epoch: 6 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.56091: 100%|██████████| 96/96 [00:05<00:00, 17.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 26.95%\n",
      "*** Epoch: 7 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.49948: 100%|██████████| 96/96 [00:06<00:00, 15.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 27.93%\n",
      "*** Epoch: 8 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.46009: 100%|██████████| 96/96 [00:05<00:00, 17.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 29.88%\n",
      "*** Epoch: 9 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.07720: 100%|██████████| 96/96 [00:05<00:00, 16.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 27.73%\n",
      "*** Epoch: 10 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.15141: 100%|██████████| 96/96 [00:06<00:00, 15.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 27.54%\n",
      "*** Epoch: 11 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.30052: 100%|██████████| 96/96 [00:05<00:00, 17.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 31.45%\n",
      "*** Epoch: 12 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.16131: 100%|██████████| 96/96 [00:06<00:00, 15.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 32.42%\n",
      "*** Epoch: 13 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.18549: 100%|██████████| 96/96 [00:05<00:00, 17.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 34.38%\n",
      "*** Epoch: 14 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.02691: 100%|██████████| 96/96 [00:06<00:00, 15.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 35.94%\n",
      "*** Epoch: 15 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.96658: 100%|██████████| 96/96 [00:05<00:00, 17.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 34.96%\n",
      "*** Epoch: 16 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.03601: 100%|██████████| 96/96 [00:05<00:00, 16.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 34.77%\n",
      "*** Epoch: 17 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.87167: 100%|██████████| 96/96 [00:06<00:00, 15.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 33.79%\n",
      "*** Epoch: 18 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.77991: 100%|██████████| 96/96 [00:05<00:00, 17.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 35.16%\n",
      "*** Epoch: 19 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 4.17713: 100%|██████████| 96/96 [00:06<00:00, 15.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 35.35%\n",
      "*** Epoch: 20 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.72587: 100%|██████████| 96/96 [00:05<00:00, 17.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 37.89%\n",
      "*** Epoch: 21 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.65346: 100%|██████████| 96/96 [00:06<00:00, 15.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 35.74%\n",
      "*** Epoch: 22 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.63766: 100%|██████████| 96/96 [00:05<00:00, 17.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 35.35%\n",
      "*** Epoch: 23 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.71225: 100%|██████████| 96/96 [00:06<00:00, 14.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 39.26%\n",
      "*** Epoch: 24 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.65176: 100%|██████████| 96/96 [00:06<00:00, 15.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 38.67%\n",
      "*** Epoch: 25 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.76204: 100%|██████████| 96/96 [00:05<00:00, 17.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 36.52%\n",
      "*** Epoch: 26 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.59846: 100%|██████████| 96/96 [00:06<00:00, 15.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 38.87%\n",
      "*** Epoch: 27 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.64759: 100%|██████████| 96/96 [00:05<00:00, 17.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 37.11%\n",
      "*** Epoch: 28 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.18299: 100%|██████████| 96/96 [00:06<00:00, 15.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 37.89%\n",
      "*** Epoch: 29 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.25882: 100%|██████████| 96/96 [00:05<00:00, 16.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 38.09%\n",
      "*** Epoch: 30 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.07264: 100%|██████████| 96/96 [00:05<00:00, 16.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 38.28%\n",
      "*** Epoch: 31 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.21905: 100%|██████████| 96/96 [00:06<00:00, 15.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 38.67%\n",
      "*** Epoch: 32 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.09636: 100%|██████████| 96/96 [00:05<00:00, 17.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 39.84%\n",
      "*** Epoch: 33 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.14585: 100%|██████████| 96/96 [00:06<00:00, 15.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 41.41%\n",
      "*** Epoch: 34 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.92465: 100%|██████████| 96/96 [00:05<00:00, 17.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 39.26%\n",
      "*** Epoch: 35 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.78828: 100%|██████████| 96/96 [00:06<00:00, 15.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 37.30%\n",
      "*** Epoch: 36 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 3.11424: 100%|██████████| 96/96 [00:05<00:00, 16.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 40.23%\n",
      "*** Epoch: 37 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.64586: 100%|██████████| 96/96 [00:05<00:00, 17.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 39.26%\n",
      "*** Epoch: 38 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.23991: 100%|██████████| 96/96 [00:06<00:00, 15.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 39.06%\n",
      "*** Epoch: 39 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.53571: 100%|██████████| 96/96 [00:05<00:00, 17.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 40.62%\n",
      "*** Epoch: 40 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.66651: 100%|██████████| 96/96 [00:06<00:00, 15.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 41.41%\n",
      "*** Epoch: 41 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.56639: 100%|██████████| 96/96 [00:05<00:00, 17.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 41.02%\n",
      "*** Epoch: 42 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.21237: 100%|██████████| 96/96 [00:06<00:00, 15.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 41.60%\n",
      "*** Epoch: 43 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.16795: 100%|██████████| 96/96 [00:05<00:00, 16.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 39.65%\n",
      "*** Epoch: 44 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.37256: 100%|██████████| 96/96 [00:06<00:00, 15.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 40.23%\n",
      "*** Epoch: 45 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.92802: 100%|██████████| 96/96 [00:06<00:00, 15.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 39.06%\n",
      "*** Epoch: 46 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.07715: 100%|██████████| 96/96 [00:05<00:00, 17.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 41.02%\n",
      "*** Epoch: 47 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.37765: 100%|██████████| 96/96 [00:06<00:00, 15.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 40.04%\n",
      "*** Epoch: 48 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.92442: 100%|██████████| 96/96 [00:05<00:00, 17.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 42.38%\n",
      "*** Epoch: 49 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 1.86587: 100%|██████████| 96/96 [00:05<00:00, 16.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 41.02%\n",
      "*** Epoch: 50 ***\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loss: 2.01111: 100%|██████████| 96/96 [00:06<00:00, 15.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy:- Teacher: 80.47%; Student: 40.04%\n",
      "\n",
      "Student Model Performance after distillation:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:00<00:00, 33.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      "[[32  3  4  1  1  2  0  1  4  4]\n",
      " [ 1 35  2  0  0  0  3  0  2 14]\n",
      " [ 6  0 16  9  7  8  5  2  2  1]\n",
      " [ 1  0  9 10  5 17  7  5  1  0]\n",
      " [ 4  0  7  0 17  1  6  6  3  4]\n",
      " [ 1  0  5  8  4 16  2  3  1  3]\n",
      " [ 0  2  8  3  5  2 17  2  0  1]\n",
      " [ 0  0  6  2 15  4  3 23  0  2]\n",
      " [13  1  1  0  0  2  0  0 29  5]\n",
      " [ 5 14  2  0  1  0  4  4  3 22]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.51      0.62      0.56        52\n",
      "           1       0.64      0.61      0.62        57\n",
      "           2       0.27      0.29      0.28        56\n",
      "           3       0.30      0.18      0.23        55\n",
      "           4       0.31      0.35      0.33        48\n",
      "           5       0.31      0.37      0.34        43\n",
      "           6       0.36      0.42      0.39        40\n",
      "           7       0.50      0.42      0.46        55\n",
      "           8       0.64      0.57      0.60        51\n",
      "           9       0.39      0.40      0.40        55\n",
      "\n",
      "    accuracy                           0.42       512\n",
      "   macro avg       0.42      0.42      0.42       512\n",
      "weighted avg       0.43      0.42      0.42       512\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "  train_data, val_data, test_data = load_dataset()\n",
    "  print(\"Train, validation, and test data size:\")\n",
    "  print(len(train_data), len(val_data), len(test_data))\n",
    "  train_loader, val_loader, test_loader = prepare_data(train_data, val_data, test_data)\n",
    "  print(\"Train, validation, and test data loader size:\")\n",
    "  print(len(train_loader), len(val_loader), len(test_loader))\n",
    "\n",
    "  teacher_model = load_model()\n",
    "  student_model = load_model(pretrained=False,width_mult=0.25)\n",
    "\n",
    "  print(\"Initial teacher Model Performance:\")\n",
    "  evaluate_model(teacher_model, val_loader)\n",
    "  print(\"\\nInitial Student Model Perfomance:\")\n",
    "  evaluate_model(student_model, val_loader)\n",
    "\n",
    "  print(\"\\nTraining Teacher Model:\")\n",
    "  teacher_model = train_model(teacher_model, train_loader, val_loader, 10)\n",
    "  print(\"\\nTeacher Model Performance after training:\")\n",
    "  evaluate_model(teacher_model, val_loader)\n",
    "\n",
    "  print(\"\\nTraining Student Model:\")\n",
    "  student_model = train_model(student_model, train_loader, val_loader, 50)\n",
    "  print(\"\\nStudent Model Performance after training:\")\n",
    "  evaluate_model(student_model, val_loader)\n",
    "\n",
    "  print(\"\\nDistillation:\")\n",
    "  student_model_distilled = load_model(pretrained=False, width_mult=0.25)\n",
    "  distilled_model = model_distillation(teacher_model, student_model_distilled, train_loader, val_loader, 50)\n",
    "  print(\"\\nStudent Model Performance after distillation:\")\n",
    "  evaluate_model(distilled_model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yj1eRoW8KrtM"
   },
   "outputs": [],
   "source": [
    "distilled_model_2 = model_distillation(teacher_model, distilled_model, train_loader, val_loader, 50)\n",
    "print(\"\\nStudent Model Performance before 2nd distillation loop:\")\n",
    "evaluate_model(distilled_model, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0a1pKEmvMVPj",
    "outputId": "c003afdf-55bb-4511-c6c1-b4f2580ad859"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Student Model Performance after 2nd distillation loop:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:00<00:00, 32.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      "[[25  6 12  1  0  1  0  0  5  2]\n",
      " [ 1 43  2  2  1  1  3  0  2  2]\n",
      " [ 4  1 22  7  4  7  7  2  1  1]\n",
      " [ 1  1 10 16  8 10  7  1  0  1]\n",
      " [ 2  0  8  4 16  7  1  6  3  1]\n",
      " [ 0  0  6  5  3 22  6  0  1  0]\n",
      " [ 0  1  5  2  2  3 24  2  1  0]\n",
      " [ 0  0  6  3 14  6  2 20  0  4]\n",
      " [11  2  1  5  0  0  1  0 30  1]\n",
      " [ 4 20  5  0  0  1  6  0  3 16]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.52      0.48      0.50        52\n",
      "           1       0.58      0.75      0.66        57\n",
      "           2       0.29      0.39      0.33        56\n",
      "           3       0.36      0.29      0.32        55\n",
      "           4       0.33      0.33      0.33        48\n",
      "           5       0.38      0.51      0.44        43\n",
      "           6       0.42      0.60      0.49        40\n",
      "           7       0.65      0.36      0.47        55\n",
      "           8       0.65      0.59      0.62        51\n",
      "           9       0.57      0.29      0.39        55\n",
      "\n",
      "    accuracy                           0.46       512\n",
      "   macro avg       0.47      0.46      0.45       512\n",
      "weighted avg       0.48      0.46      0.45       512\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nStudent Model Performance after 2nd distillation loop:\")\n",
    "evaluate_model(distilled_model_2, val_loader)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}