Facial Expression Recognition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align=center> Facial Expression Recognition Classifier Model</h1>\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Libraries\n",
    "- Let us import all the required Libraries and Packages including Tensorflow\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wvGxjjeV-9Ls"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import utils\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# To set the backend of matplotlib to the 'inline' backend\n",
    "%matplotlib inline\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.utils import plot_model\n",
    "from tensorflow.keras.models import Model, Sequential\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "from tensorflow.keras.layers import Dense, Input, Dropout,Flatten, Conv2D\n",
    "from tensorflow.keras.layers import BatchNormalization, Activation, MaxPooling2D\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Sample Images\n",
    "- Let us have a look at random images from the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.datasets.fer.plot_example_images(plt).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Sets\n",
    "- Let us look at the number of Images in Training and Testing Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 139
    },
    "colab_type": "code",
    "id": "TalL_1Qr-9Qz",
    "outputId": "f5fb9b05-976a-4979-ea23-33c3d87efb94"
   },
   "outputs": [],
   "source": [
    "\n",
    "print(\"Images in Training Data\")\n",
    "print(\"_______________________\")\n",
    "for expression in os.listdir(\"training_data/\"):\n",
    "    print(str(len(os.listdir(\"training_data/\" + expression))) + \" \" + expression + \" images\")\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "print(\"Images in Testing Data\")\n",
    "print(\"_______________________\")\n",
    "for expression in os.listdir(\"testing_data/\"):\n",
    "    print(str(len(os.listdir(\"testing_data/\" + expression))) + \" \" + expression + \" images\")\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Training and Validation Batches\n",
    "- Using the ImageDataGenerators Let us create Training and Validation Batches by loading images from corresponding directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "iri8ehFw-9Tj",
    "outputId": "1cff3826-c0a9-41ff-a61b-5a677de101ae"
   },
   "outputs": [],
   "source": [
    "img_size = 48\n",
    "batch_size = 64\n",
    "\n",
    "datagen_train = ImageDataGenerator(horizontal_flip=True)\n",
    "train_generator = datagen_train.flow_from_directory(\"training_data/\",\n",
    "                                                    target_size=(img_size,img_size),\n",
    "                                                    color_mode=\"grayscale\",\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    class_mode='categorical',\n",
    "                                                    shuffle=True)\n",
    "\n",
    "datagen_validation = ImageDataGenerator(horizontal_flip=True)\n",
    "validation_generator = datagen_validation.flow_from_directory(\"testing_data/\",\n",
    "                                                    target_size=(img_size,img_size),\n",
    "                                                    color_mode=\"grayscale\",\n",
    "                                                    batch_size=batch_size,\n",
    "                                                    class_mode='categorical',\n",
    "                                                    shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Convolutional Nueral Network (CNN) Model\n",
    "- Let us create a Nueral Network using 4 Convolutional Layers and 2 Fully Connected dense Layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialising the CNN\n",
    "model = Sequential()\n",
    "\n",
    "# 1st Convolution Layer\n",
    "\n",
    "# There are 64 (3,3) filters with \"same\" Padding and Shape of the Input_Image is (48,48,1)\n",
    "model.add(Conv2D(64,(3,3), padding='same', input_shape=(48, 48,1)))\n",
    "\n",
    "# Normalizing to speed up learning.\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# Applying Non Linear Activation Function \"relu\"\n",
    "model.add(Activation('relu'))  \n",
    "\n",
    "# Adding a Max Pool Layer of size (2,2)\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# Dropout layer with 0.25 fraction of the input units to drop\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# 2nd Convolution layer\n",
    "\n",
    "# There are 128 (5,5) filters with \"same\" Padding \n",
    "model.add(Conv2D(128,(5,5), padding='same'))\n",
    "\n",
    "# Normalizing to speed up learning.\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# Applying Non Linear Activation Function \"relu\"\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "# Adding a Max Pool Layer of size (2,2)\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# Dropout layer with 0.25 fraction of the input units to drop\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# 3rd Convolution layer\n",
    "\n",
    "# There are 512 (3,3) filters with \"same\" Padding \n",
    "\n",
    "model.add(Conv2D(512,(3,3), padding='same'))\n",
    "\n",
    "# Normalizing to speed up learning.\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# Applying Non Linear Activation Function \"relu\"\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "# Adding a Max Pool Layer of size (2,2)\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# Dropout layer with 0.25 fraction of the input units to drop\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# 4th Convolution layer\n",
    "\n",
    "# There are 512 (3,3) filters with \"same\" Padding \n",
    "model.add(Conv2D(512,(3,3), padding='same'))\n",
    "\n",
    "# Normalizing to speed up learning.\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# Applying Non Linear Activation Function \"relu\"\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "# Adding a Max Pool Layer of size (2,2)\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# Dropout layer with 0.25 fraction of the input units to drop \n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "\n",
    "\n",
    "# Flattening\n",
    "model.add(Flatten())\n",
    "\n",
    "\n",
    "# Fully connected layer with 256 nuerons\n",
    "model.add(Dense(256))\n",
    "\n",
    "# Normalizing to speed up learning.\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# Applying Non Linear Activation Function \"relu\"\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "# Dropout layer with 0.25 fraction of the input units to drop\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "\n",
    "\n",
    "# Fully connected layer with 512 nuerons\n",
    "model.add(Dense(512))\n",
    "\n",
    "# Normalizing to speed up learning.\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# Applying Non Linear Activation Function \"relu\"\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "# Dropout layer with 0.25 fraction of the input units to drop\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "# Adding a final Dense Layer with 7 outputs corresponding to 7 different emotions with a \"softmax\" Activation Function \n",
    "model.add(Dense(7, activation='softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compiling the Model\n",
    "- Let us use Adam Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adam is an optimization algorithm that can be used instead of the classical stochastic gradient descent \n",
    "# procedure to update network weights iterative based in training data.\n",
    "\n",
    "# Let us choose a Learning rate of 0.0005 \n",
    "opt = Adam(lr=0.0005)\n",
    "\n",
    "\n",
    "# Compile defines the loss function, the optimizer and the metrics.\n",
    "\n",
    "# As we have Categorical Values we will use 'categorical_crossentropy' loss function\n",
    "model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Let us check the details of the Model\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train and Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let us train the Model 15 times\n",
    "epochs = 15\n",
    "\n",
    "steps_per_epoch = train_generator.n//train_generator.batch_size\n",
    "validation_steps = validation_generator.n//validation_generator.batch_size\n",
    "\n",
    "# Create a Callback which reduces the Learning rate by a factor of \"0.1\" when the val_loss does not decrease\n",
    "# after 2 epochs also and allowing the minimum value of Learning Rate to be 0.00001\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1,\n",
    "                              patience=2, min_lr=0.00001, mode='auto')\n",
    "\n",
    "\n",
    "# Create another Callback which saves the Model Weights by monitoring the Val_Accuracy\n",
    "checkpoint = ModelCheckpoint(\"model_weights.h5\", monitor='val_accuracy',\n",
    "                             save_weights_only=True, mode='max', verbose=1)\n",
    "\n",
    "\n",
    "# A callback is an object that can perform actions at various stages of training\n",
    "# (e.g. at the start or end of an epoch, before or after a single batch, etc).\n",
    "callbacks = [checkpoint, reduce_lr]\n",
    "\n",
    "# Fitting the model .\n",
    "history = model.fit(\n",
    "    x=train_generator,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    epochs=epochs,\n",
    "    validation_data = validation_generator,\n",
    "    validation_steps = validation_steps,\n",
    "    callbacks=callbacks\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Represent Model as JSON String"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cHw8ir7CVAE0"
   },
   "outputs": [],
   "source": [
    "# Converting the model into JSON format and storing it in \"fer_model.json\" file. \n",
    "model_json = model.to_json()\n",
    "with open(\"model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Facial_Expression_Training.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}