Sentiment_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sentiment Analysis",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E3ajeDAljwPL",
        "colab_type": "text"
      },
      "source": [
        "# Sentiment Analysis"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "55octSxkrZLu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torchtext import data\n",
        "from torchtext import datasets\n",
        "import random"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CjctfamHkToi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "SEED = 12\n",
        "\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "TEXT = data.Field(tokenize = 'spacy', include_lengths = True)\n",
        "LABEL = data.LabelField(dtype = torch.float)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "EumH1qRfkd67"
      },
      "source": [
        "## Importing dataset\n",
        "- [IMDB dataset](https://torchtext.readthedocs.io/en/latest/datasets.html#)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TKezPnoMrgqN",
        "colab_type": "code",
        "outputId": "d0932306-8a65-471e-f1c5-093c6d6f9fbf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "downloading aclImdb_v1.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "aclImdb_v1.tar.gz: 100%|██████████| 84.1M/84.1M [00:07<00:00, 10.9MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "079_F3J0l7pE",
        "colab_type": "text"
      },
      "source": [
        "- Splitting dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MluAGdd0rj5F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_data, valid_data = train_data.split(random_state = random.seed(SEED))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H8ZmZCm5rnnt",
        "colab_type": "code",
        "outputId": "29ce85cf-f6ca-4152-8258-2f7b92e75fca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "MAX_VOCAB_SIZE = 25_000 # taking only 25000 words based frequency of them occuring in the dataset\n",
        "\n",
        "TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE, vectors = \"glove.6B.100d\", unk_init = torch.Tensor.normal_)\n",
        "LABEL.build_vocab(train_data)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".vector_cache/glove.6B.zip: 862MB [06:30, 2.21MB/s]                          \n",
            "100%|█████████▉| 399164/400000 [00:19<00:00, 20522.49it/s]"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gs21Jh-FrtIu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "# spliting and sort data within the batch\n",
        "# batches of sentences with  similar lengths together\n",
        "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits((train_data, valid_data, test_data),batch_size = BATCH_SIZE,sort_within_batch = True,device = device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rU_bGc5bsnmd",
        "colab_type": "code",
        "outputId": "747f1502-adc2-4365-fbd6-e9e121cd03e2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(device)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dQd87jngkXU0",
        "colab_type": "text"
      },
      "source": [
        "# Implementation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHWdlqZysp9e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class RNN(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, \n",
        "                 bidirectional, dropout, pad_idx):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)#initializes pads to zeros\n",
        "        \n",
        "        # n_layers of LSTM's stacked together\n",
        "        self.rnn = nn.LSTM(embedding_dim, hidden_dim,num_layers=n_layers,bidirectional=bidirectional,dropout=dropout)\n",
        "        \n",
        "        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, text, text_lengths):\n",
        "        embedded = self.dropout(self.embedding(text))\n",
        "        #Packs a Tensor containing padded sequences of variable length.\n",
        "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths)\n",
        "        \n",
        "        packed_output, (hidden, cell) = self.rnn(packed_embedded)\n",
        "        \n",
        "        #unpacking the pad packed sequence\n",
        "        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
        "        \n",
        "        #concat the final forward (hidden[-2,:,:]) and backward (hidden[-1,:,:]) hidden layers and applying dropout\n",
        "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
        "                \n",
        "        #hidden = [batch size, hid dim * num directions]\n",
        "            \n",
        "        return self.fc(hidden)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LCnQ21EMsyuY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "INPUT_DIM = len(TEXT.vocab)\n",
        "EMBEDDING_DIM = 100\n",
        "HIDDEN_DIM = 256\n",
        "OUTPUT_DIM = 1\n",
        "N_LAYERS = 2\n",
        "BIDIRECTIONAL = True\n",
        "DROPOUT = 0.5\n",
        "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token] # indexing the padded tokens\n",
        "\n",
        "model = RNN(INPUT_DIM, \n",
        "            EMBEDDING_DIM, \n",
        "            HIDDEN_DIM, \n",
        "            OUTPUT_DIM, \n",
        "            N_LAYERS, \n",
        "            BIDIRECTIONAL, \n",
        "            DROPOUT, \n",
        "            PAD_IDX)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7qvqDaEtlxzz",
        "colab_type": "text"
      },
      "source": [
        "- replace the initial weights of the embedding layer with the pre-trained embeddings."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zLv0wvbYtDZ_",
        "colab_type": "code",
        "outputId": "10be1663-e9d8-4ea1-e6da-7b67e1c0bd5f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134
        }
      },
      "source": [
        "pretrained_embeddings = TEXT.vocab.vectors\n",
        "model.embedding.weight.data.copy_(pretrained_embeddings)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.1320, -0.1254,  0.3443,  ...,  1.6690, -0.1693,  0.2577],\n",
              "        [ 1.1629, -0.1765,  0.8580,  ..., -0.7312, -0.3655,  0.9063],\n",
              "        [-0.0382, -0.2449,  0.7281,  ..., -0.1459,  0.8278,  0.2706],\n",
              "        ...,\n",
              "        [-0.4625, -0.1248, -0.3915,  ..., -0.7118,  0.0469,  0.4989],\n",
              "        [ 0.0119,  0.6618,  0.4938,  ..., -0.2238, -0.2141,  0.5771],\n",
              "        [-0.8513, -0.5600,  0.0165,  ...,  1.5267,  0.2747,  0.6986]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RzXwjFPwlYXR",
        "colab_type": "text"
      },
      "source": [
        "- As our unk and pad token aren't in the pre-trained vocabulary they have been initialized using unk_init (an $\\mathcal{N}(0,1)$ distribution) when building our vocab."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQvmicSbtK5M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
        "\n",
        "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
        "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qIOqkYPRtNpZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "optimizer = optim.Adam(model.parameters())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1JqONPrVtQC9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "criterion = nn.BCEWithLogitsLoss()\n",
        "\n",
        "model = model.to(device)\n",
        "criterion = criterion.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L7W-ynYLtSc9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def accuracy(preds, y):\n",
        "\n",
        "    #round predictions to the closest integer\n",
        "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = (rounded_preds == y).float() #convert into float for division \n",
        "    acc = correct.sum() / len(correct)\n",
        "    return acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pmjxw193tV-K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(model, iterator, optimizer, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    for batch in iterator:\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        text, text_lengths = batch.text\n",
        "        \n",
        "        predictions = model(text, text_lengths).squeeze(1)\n",
        "        \n",
        "        loss = criterion(predictions, batch.label)\n",
        "        \n",
        "        acc = accuracy(predictions, batch.label)\n",
        "        \n",
        "        loss.backward()\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lQoqeVXqtZf0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.eval() #to ensure the dropout is \"turned off\" while evaluating\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in iterator:\n",
        "\n",
        "            text, text_lengths = batch.text\n",
        "            \n",
        "            predictions = model(text, text_lengths).squeeze(1)\n",
        "            \n",
        "            loss = criterion(predictions, batch.label)\n",
        "            \n",
        "            acc = accuracy(predictions, batch.label)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GBRh30yztcqz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jIP41FeetfQH",
        "colab_type": "code",
        "outputId": "a0c88291-df44-437b-acbb-53c29947cb2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "N_EPOCHS = 5\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "\n",
        "    start_time = time.time()\n",
        "    \n",
        "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
        "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
        "    \n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'weights.pt')\n",
        "    \n",
        "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
        "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01 | Epoch Time: 1m 37s\n",
            "\tTrain Loss: 0.653 | Train Acc: 60.75%\n",
            "\t Val. Loss: 0.570 |  Val. Acc: 70.55%\n",
            "Epoch: 02 | Epoch Time: 1m 37s\n",
            "\tTrain Loss: 0.641 | Train Acc: 63.31%\n",
            "\t Val. Loss: 0.673 |  Val. Acc: 68.90%\n",
            "Epoch: 03 | Epoch Time: 1m 37s\n",
            "\tTrain Loss: 0.540 | Train Acc: 72.59%\n",
            "\t Val. Loss: 0.395 |  Val. Acc: 82.88%\n",
            "Epoch: 04 | Epoch Time: 1m 38s\n",
            "\tTrain Loss: 0.373 | Train Acc: 84.25%\n",
            "\t Val. Loss: 0.332 |  Val. Acc: 86.37%\n",
            "Epoch: 05 | Epoch Time: 1m 37s\n",
            "\tTrain Loss: 0.317 | Train Acc: 87.23%\n",
            "\t Val. Loss: 0.305 |  Val. Acc: 87.70%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dCPn_mTltj-h",
        "colab_type": "code",
        "outputId": "c3016adf-560f-4340-bbcb-1e71a6c0b78f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.load_state_dict(torch.load('weights.pt'))\n",
        "\n",
        "test_loss, test_acc = evaluate(model, test_iterator, criterion)\n",
        "\n",
        "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Loss: 0.324 | Test Acc: 86.61%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uX4Z_HIUtr7I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "nlp = spacy.load('en') # loading spaCy english model\n",
        "\n",
        "def predict_sentiment(model, sentence):\n",
        "    model.eval()\n",
        "    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]\n",
        "    indexed = [TEXT.vocab.stoi[t] for t in tokenized]\n",
        "    length = [len(indexed)]\n",
        "    tensor = torch.LongTensor(indexed).to(device)\n",
        "    tensor = tensor.unsqueeze(1)\n",
        "    length_tensor = torch.LongTensor(length)\n",
        "    prediction = torch.sigmoid(model(tensor, length_tensor))\n",
        "    return prediction.item()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yegLni7sJ85J",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "56b5aa42-5960-4f55-8a7f-7dc15749bb5b"
      },
      "source": [
        "predict_sentiment(model, \"This movie is piece of shit\")"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.17475979030132294"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LHqZSL74REfN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a8269513-542e-4336-e2e5-d93c072d84f0"
      },
      "source": [
        "predict_sentiment(model, \"This film is total crap\")"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9748067855834961"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    }
  ]
}