fairness_classification/rnn_models/Classification_RoBERTa_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "268d41ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dependencies to run the notebook\n",
    "\n",
    "# !pip install torch==1.12.1\n",
    "# !pip install torchmetrics==0.10.2\n",
    "# !pip install torchvision==0.14.0\n",
    "# !pip install texttable==1.6.4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61dd939a",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"3\">Download the below files from https://drive.google.com/drive/folders/1q50QMurzK9a5l4JBHWjf8VuWcZkbF7PM to run this notebook : <br>\n",
    "1) train_roberta_embeddings.pkl <br>\n",
    "2) test_roberta_embeddings.pkl <br> </font>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "0fef83eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from torchvision import datasets, transforms\n",
    "from torchvision.transforms import ToTensor\n",
    "from torch.autograd import Variable\n",
    "import torch.optim as optim\n",
    "import warnings\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "import pandas as pd\n",
    "from torchmetrics import Accuracy, F1Score, Precision, Recall\n",
    "from texttable import Texttable\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "warnings.filterwarnings('ignore')\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "698d71d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4bf44010",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read train BERT embeddings\n",
    "with open('../../../../../train_roberta_embeddings.pkl', 'rb') as f:\n",
    "    training_data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7e45b2ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read test BERT embeddings\n",
    "with open('../../../../../test_roberta_embeddings.pkl', 'rb') as f:\n",
    "    testing_data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6c2bd91a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The data is :  embeddings\n",
      "The data is :  tokenized_txt\n"
     ]
    }
   ],
   "source": [
    "# check\n",
    "for item in training_data:\n",
    "    print('The data is : ', item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c5dd710e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ġcontent',\n",
       " 'Ġlicense',\n",
       " 'Ġand',\n",
       " 'Ġintellectual',\n",
       " 'Ġproperty',\n",
       " 'Ġrights',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check\n",
    "training_data['tokenized_txt'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dca0767b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7531, torch.Size([1, 512, 768]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(training_data['embeddings']), training_data['embeddings'][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "14c524e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7531, 512)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(training_data['tokenized_txt']) , len(training_data['tokenized_txt'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1481e82e",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"4\">Get the labels from train and test files.</font>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7ec7097e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset_path = \"../legal_bert/data/tos_clauses_train.csv\"\n",
    "test_dataset_path = \"../legal_bert/data/tos_clauses_dev.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7feb1a37",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(train_dataset_path, header=0)\n",
    "test_df = pd.read_csv(test_dataset_path, header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8a29d269",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_targets = train_df.label.values\n",
    "test_targets = test_df.label.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "541edf2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 1, 0])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0378fa66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on GPU.\n"
     ]
    }
   ],
   "source": [
    "# First checking if GPU is available\n",
    "train_on_gpu=torch.backends.mps.is_available()\n",
    "if(train_on_gpu):\n",
    "    print('Training on GPU.')\n",
    "    device = torch.device(\"mps\")\n",
    "else:\n",
    "    print('No GPU available, training on CPU.')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f92206a0",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"4\">Create Dataset, Train and Test Classes</font>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "efaf4ad4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(object):\n",
    "    \"\"\"An abstract class representing a Dataset.\n",
    "    All other datasets should subclass it. All subclasses should\n",
    "    override ``__len__``, that provides the size of the dataset,\n",
    "    and ``__getitem__``, supporting integer indexing in range\n",
    "    from 0 to len(self) exclusive.\n",
    "    \"\"\"\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def __len__(self):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def __add__(self, other):\n",
    "        return ConcatDataset([self, other])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2bb78e49",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Train_Model(Dataset):\n",
    "    \n",
    "    def __init__(self, X, Y, transform=None):\n",
    "        self.data1 = X\n",
    "        self.data2 = Y\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.data1)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        x = self.data1[index]\n",
    "        y = self.data2[index]\n",
    "        \n",
    "        \n",
    "        if self.transform is not None:\n",
    "            x = torch.tensor(x)\n",
    "            \n",
    "        return x, torch.tensor(y)                   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "92f16d30",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Test_Model(Dataset):\n",
    "    \n",
    "    def __init__(self, X, transform=None):\n",
    "        self.data1 = X\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.data1)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        x = self.data1[index]\n",
    "                \n",
    "        if self.transform is not None:\n",
    "            x = torch.tensor(x)\n",
    "            \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7d37e581",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<torch.utils.data.sampler.SubsetRandomSampler object at 0x157348b80>\n",
      "train_fair:6705\n",
      "train_unfair:826\n"
     ]
    }
   ],
   "source": [
    "test_len = len(test_df)\n",
    "train_len = len(train_df)\n",
    "X_train_tensor = Train_Model(train_df['sentences'], train_df['label'])\n",
    "# X_test_tensor = Train_Model(test_df)\n",
    "\n",
    "num_train = len(X_train_tensor)\n",
    "indices = list(range(num_train))\n",
    "np.random.shuffle(indices)\n",
    "# split = int(np.floor(num_train))\n",
    "# train_idx = indices[split:]\n",
    "\n",
    "train_sampler = SubsetRandomSampler(indices)\n",
    "# valid_sampler = SubsetRandomSampler(valid_idx)\n",
    "train_df_by_index = train_df.loc[indices]\n",
    "# val_df_by_index = df_train.loc[valid_idx]\n",
    "train_fair = sum(train_df_by_index['label'] == 0)\n",
    "train_unfair = sum(train_df_by_index['label'] == 1)\n",
    "# val_fair = sum(val_df_by_index['label'] == 0)\n",
    "# val_unfair = sum(val_df_by_index['label'] == 1)\n",
    "\n",
    "print(\"train_fair:\" + str(train_fair))\n",
    "print(\"train_unfair:\" + str(train_unfair))\n",
    "# print(\"val_fair:\" + str(val_fair))\n",
    "# print(\"val_unfair:\" + str(val_unfair))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cca2a92d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = Train_Model(training_data['embeddings'],\n",
    "                              train_targets,\n",
    "                              transform=transforms.ToTensor())\n",
    "test_data = Test_Model(testing_data['embeddings'],\n",
    "                            transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "046b7413",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"4\">Prepare Data loaders</font>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "14d9572e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of subprocesses to use for data loading\n",
    "num_workers = 0\n",
    "# how many samples per batch to load\n",
    "batch_size = 20\n",
    "\n",
    "# prepare data loaders\n",
    "train_loader = torch.utils.data.DataLoader(train_data,\n",
    "                                             batch_size=batch_size,\n",
    "                                             sampler=train_sampler,\n",
    "                                             num_workers=num_workers)\n",
    "test_loader = torch.utils.data.DataLoader(test_data,\n",
    "                                            batch_size=batch_size,\n",
    "                                            num_workers=num_workers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "36811bd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample input size:  torch.Size([20, 1, 512, 768])\n",
      "Sample input: \n",
      " tensor([[[[ 0.0066,  0.0683, -0.0492,  ..., -0.1372, -0.0292,  0.0263],\n",
      "          [-0.2287,  0.0436,  0.2377,  ...,  0.3928,  0.0439, -0.1978],\n",
      "          [-0.0615,  0.0450,  0.0209,  ...,  0.1682,  0.1016,  0.1232],\n",
      "          ...,\n",
      "          [ 0.1794,  0.1406, -0.0955,  ...,  0.0755, -0.0396, -0.0895],\n",
      "          [ 0.0464,  0.1520, -0.2502,  ...,  0.0440, -0.1282,  0.0411],\n",
      "          [ 0.0203,  0.0635, -0.1043,  ..., -0.2142, -0.0215,  0.0154]]],\n",
      "\n",
      "\n",
      "        [[[-0.0157,  0.0656, -0.0627,  ..., -0.1670, -0.0277,  0.0606],\n",
      "          [ 0.1689,  0.2124, -0.0917,  ...,  0.2360, -0.1986,  0.0415],\n",
      "          [ 0.0319,  0.1823, -0.1980,  ...,  0.5952,  0.2550,  0.0426],\n",
      "          ...,\n",
      "          [ 0.1710,  0.1172, -0.0953,  ..., -0.1375, -0.0492, -0.0842],\n",
      "          [-0.0206,  0.0414, -0.2403,  ...,  0.1024, -0.1325, -0.0109],\n",
      "          [-0.0103,  0.0523, -0.1137,  ..., -0.2497, -0.0230,  0.0599]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0226,  0.0603, -0.0450,  ..., -0.0842, -0.0374,  0.0268],\n",
      "          [ 0.0968, -0.1603, -0.0780,  ..., -0.2985,  0.0828, -0.1763],\n",
      "          [ 0.3798,  0.0613, -0.2693,  ..., -0.1342, -0.0473,  0.2062],\n",
      "          ...,\n",
      "          [ 0.1877,  0.0892, -0.1397,  ..., -0.0111, -0.0837, -0.0649],\n",
      "          [ 0.0776,  0.0496, -0.2081,  ...,  0.0558, -0.1779,  0.0325],\n",
      "          [ 0.0415,  0.0525, -0.0868,  ..., -0.1565, -0.0251,  0.0045]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[ 0.0021,  0.0398, -0.0464,  ..., -0.0867, -0.0450, -0.0297],\n",
      "          [-0.1974, -0.1885,  0.0993,  ...,  0.0230,  0.1897, -0.3832],\n",
      "          [-0.1851, -0.1994, -0.0140,  ..., -0.3976, -0.0954, -0.1515],\n",
      "          ...,\n",
      "          [ 0.1764,  0.1096, -0.1063,  ...,  0.0395, -0.0947, -0.1005],\n",
      "          [ 0.0149,  0.0753, -0.2604,  ...,  0.2026, -0.1941,  0.0066],\n",
      "          [ 0.0068,  0.0219, -0.0986,  ..., -0.1577, -0.0236, -0.0414]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0013,  0.0748, -0.0577,  ..., -0.1123, -0.0302,  0.0073],\n",
      "          [ 0.2038,  0.2836, -0.1175,  ...,  0.1780, -0.0165,  0.0336],\n",
      "          [-0.2118,  0.1327,  0.3428,  ...,  0.4540,  0.1634, -0.2084],\n",
      "          ...,\n",
      "          [ 0.1602,  0.1302, -0.0810,  ...,  0.0460, -0.0285, -0.0881],\n",
      "          [ 0.0384,  0.1558, -0.2483,  ...,  0.0729, -0.0944,  0.0357],\n",
      "          [ 0.0047,  0.0700, -0.1088,  ..., -0.1862, -0.0288,  0.0013]]],\n",
      "\n",
      "\n",
      "        [[[-0.0050,  0.0683, -0.0542,  ..., -0.0900, -0.0557,  0.0035],\n",
      "          [ 0.2363, -0.0482, -0.0272,  ..., -0.1281, -0.0155,  0.0537],\n",
      "          [-0.0744,  0.0847, -0.2169,  ...,  0.1401,  0.0314, -0.0164],\n",
      "          ...,\n",
      "          [ 0.1291,  0.1604, -0.1379,  ..., -0.0196, -0.0833, -0.1198],\n",
      "          [ 0.0842,  0.1386, -0.2248,  ...,  0.0174, -0.1137, -0.0185],\n",
      "          [ 0.0013,  0.0614, -0.0949,  ..., -0.1469, -0.0546, -0.0166]]]])\n",
      "\n",
      "Sample label size:  torch.Size([20])\n",
      "Sample label: \n",
      " tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n"
     ]
    }
   ],
   "source": [
    "# check sizes\n",
    "dataiter = iter(train_loader)\n",
    "sample_x, sample_y = dataiter.next()\n",
    "\n",
    "print('Sample input size: ', sample_x.size()) # batch_size, seq_length\n",
    "print('Sample input: \\n', sample_x)\n",
    "print()\n",
    "print('Sample label size: ', sample_y.size()) # batch_size\n",
    "print('Sample label: \\n', sample_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4ccefed",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"5\">SIMPLE RNN</font><br>\n",
    "<font size=\"2.5\">Number of hidden dimension : 20</font> <br>\n",
    "<font size=\"2.5\">Number of layers: 1</font> <br>\n",
    "<font size=\"2.5\">Number of epochs: 5</font> <br>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f35604d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNet(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        \n",
    "        super(RNNet, self).__init__()\n",
    "\n",
    "         # Number of hidden dimensions\n",
    "        self.hidden_dim = hidden_dim\n",
    "        \n",
    "        # RNN\n",
    "        self.rnn = nn.RNN(input_dim, hidden_dim, num_layers=1, batch_first=True, nonlinearity='relu')\n",
    "        \n",
    "        # Readout layer\n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        # Initialize hidden state with zeros\n",
    "        h0 = Variable(torch.zeros(1, x.size(0), self.hidden_dim))\n",
    "            \n",
    "        # One time step\n",
    "        out, hn = self.rnn(x, h0)\n",
    "        out = self.fc(out[:, -1, :]) \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d5c62989",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RNNet(\n",
      "  (rnn): RNN(768, 20, batch_first=True)\n",
      "  (fc): Linear(in_features=20, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_DIM = 768\n",
    "HIDDEN_DIM = 20\n",
    "OUTPUT_DIM = 2\n",
    "\n",
    "model_rnn = RNNet(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)\n",
    "print(model_rnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1090e624",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_RNN(model, train_loader, optimizer, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in train_loader:\n",
    "        \n",
    "        train  = Variable(batch[0].view(-1, 512, 768))\n",
    "        labels = Variable(batch[1])\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "                \n",
    "        output = model(train)\n",
    "        \n",
    "        loss = criterion(output, labels) \n",
    "        \n",
    "        acc = binary_accuracy(output.argmax(-1), labels)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(train_loader), epoch_acc / len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c4a1f44b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ac0dd4bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_accuracy(preds, y):\n",
    "    \"\"\"\n",
    "    Returns accuracy per batch\n",
    "    \"\"\"\n",
    "    correct = (preds == y).float() \n",
    "    acc = correct.sum() / len(correct)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "53e1f120",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 22s\n",
      "\tTrain Loss:     0.686 | Train Acc: 88.89%\n",
      "Epoch: 02 | Epoch Time: 0m 16s\n",
      "\tTrain Loss:     0.677 | Train Acc: 88.53%\n",
      "Epoch: 03 | Epoch Time: 0m 13s\n",
      "\tTrain Loss:     0.669 | Train Acc: 86.25%\n",
      "Epoch: 04 | Epoch Time: 0m 13s\n",
      "\tTrain Loss:     0.656 | Train Acc: 79.78%\n",
      "Epoch: 05 | Epoch Time: 0m 14s\n",
      "\tTrain Loss:     0.646 | Train Acc: 74.41%\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(weight = torch.FloatTensor([1/train_fair, 1/train_unfair]))\n",
    "# criterion = nn.NLLLoss()\n",
    "\n",
    "optimizer_rnn = optim.Adam(model_rnn.parameters(), lr=1e-4)\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    rnn_train_loss, rnn_train_acc = train_RNN(model_rnn,\n",
    "                                      train_loader,\n",
    "                                      optimizer_rnn,\n",
    "                                      criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: \\\n",
    "    {rnn_train_loss:.3f} | Train Acc: {rnn_train_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5d4ffec6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of simple RNN : 0.7419012188911438\n",
      "F1 score of simple RNN : 0.5697938203811646\n",
      "Precision of simple RNN : 0.56910240650177\n",
      "Recall of simple RNN : 0.635807991027832\n"
     ]
    }
   ],
   "source": [
    "test_loader_predict = torch.utils.data.DataLoader(\n",
    "    test_data,\n",
    "    batch_size=1,\n",
    "    num_workers=0)\n",
    "\n",
    "def predict(model, dataloader):\n",
    "    prediction_list = []\n",
    "    for i, batch in enumerate(dataloader):\n",
    "        test  = Variable(batch.view(-1, 512, 768))\n",
    "        outputs = model(test)\n",
    "        _, predicted = torch.max(outputs.data, 1) \n",
    "        prediction_list.append(predicted.cpu())\n",
    "    return prediction_list\n",
    "\n",
    "predictions_rnn = predict(model_rnn, test_loader_predict)\n",
    "\n",
    "a_tensor = torch.IntTensor(predictions_rnn)\n",
    "b_tensor = torch.IntTensor(test_targets)\n",
    "\n",
    "accuracy = Accuracy()\n",
    "rnn_test_acc = accuracy(a_tensor, b_tensor).item()\n",
    "\n",
    "f1 = F1Score(num_classes=2, average='macro') # checked if weighted can be used\n",
    "rnn_f1_score = f1(a_tensor, b_tensor).item()\n",
    "\n",
    "precision = Precision(average='macro', num_classes=2)\n",
    "rnn_precision = precision(a_tensor, b_tensor).item()\n",
    "\n",
    "recall = Recall(average='macro', num_classes=2)\n",
    "rnn_recall = recall(a_tensor, b_tensor).item()\n",
    "\n",
    "print(\"Accuracy of simple RNN :\", rnn_test_acc)\n",
    "print(\"F1 score of simple RNN :\", rnn_f1_score)\n",
    "print(\"Precision of simple RNN :\", rnn_precision)\n",
    "print(\"Recall of simple RNN :\", rnn_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0669984b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+----------------+---------------+----------+-----------+---------+\n",
      "|   Model    | Train accuracy | Test Accuracy | F1-score | Precision | Recall  |\n",
      "+============+================+===============+==========+===========+=========+\n",
      "| Simple RNN | 0.74408        | 0.74190       | 0.56979  | 0.56910   | 0.63581 |\n",
      "+------------+----------------+---------------+----------+-----------+---------+ \n",
      "\n"
     ]
    }
   ],
   "source": [
    "table = Texttable()\n",
    "table.set_cols_dtype(['a','f','f','f','f','f'])\n",
    "table.set_precision(5)\n",
    "table.add_rows([[\"Model\", \"Train accuracy\", \"Test Accuracy\", \"F1-score\", \"Precision\", \"Recall\"],\n",
    "                [\"Simple RNN\", rnn_train_acc, rnn_test_acc, rnn_f1_score, rnn_precision, rnn_recall]])\n",
    "print(table.draw(),\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfe86eec",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"5\">Gated RNN</font><br>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e40ad5f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRU_Network(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        \n",
    "        super(GRU_Network, self).__init__()\n",
    "\n",
    "         # Number of hidden dimensions\n",
    "        self.hidden_dim = hidden_dim\n",
    "        \n",
    "        # RNN\n",
    "        self.rnn = nn.GRU(input_dim, hidden_dim, num_layers=1, batch_first=True)\n",
    "        \n",
    "        # Readout layer\n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        # Initialize hidden state with zeros\n",
    "        h0 = Variable(torch.zeros(1, x.size(0), self.hidden_dim))\n",
    "            \n",
    "        # One time step\n",
    "        out, hn = self.rnn(x, h0)\n",
    "        out = self.fc(out[:, -1, :]) \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ace29e9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GRU_Network(\n",
      "  (rnn): GRU(768, 20, batch_first=True)\n",
      "  (fc): Linear(in_features=20, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model_gru = GRU_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)\n",
    "print(model_gru)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "cc5174a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_GRU(model, train_loader, optimizer, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in train_loader:\n",
    "        \n",
    "        train  = Variable(batch[0].view(-1, 512, 768))\n",
    "        labels = Variable(batch[1])\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "                \n",
    "        output = model(train)\n",
    "        \n",
    "        loss = criterion(output, labels) \n",
    "        \n",
    "        acc = binary_accuracy(output.argmax(-1), labels)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(train_loader), epoch_acc / len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ec37270a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 30s\n",
      "\tTrain Loss:     0.689 | Train Acc: 80.77%\n",
      "Epoch: 02 | Epoch Time: 0m 25s\n",
      "\tTrain Loss:     0.672 | Train Acc: 80.43%\n",
      "Epoch: 03 | Epoch Time: 0m 26s\n",
      "\tTrain Loss:     0.660 | Train Acc: 75.23%\n",
      "Epoch: 04 | Epoch Time: 0m 27s\n",
      "\tTrain Loss:     0.646 | Train Acc: 74.34%\n",
      "Epoch: 05 | Epoch Time: 0m 25s\n",
      "\tTrain Loss:     0.635 | Train Acc: 73.11%\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(weight = torch.FloatTensor([1/train_fair, 1/train_unfair]))\n",
    "# criterion = nn.NLLLoss()\n",
    "\n",
    "optimizer_gru = optim.Adam(model_gru.parameters(), lr=1e-4)\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    gru_train_loss, gru_train_acc = train_RNN(model_gru,\n",
    "                                      train_loader,\n",
    "                                      optimizer_gru,\n",
    "                                      criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: \\\n",
    "    {gru_train_loss:.3f} | Train Acc: {gru_train_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2d4fca45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Gated RNN : 0.7546468377113342\n",
      "F1 score of Gated RNN : 0.5704818964004517\n",
      "Precision of Gated RNN : 0.5669399499893188\n",
      "Recall of Gated RNN : 0.6238023042678833\n"
     ]
    }
   ],
   "source": [
    "test_loader_predict = torch.utils.data.DataLoader(\n",
    "    test_data,\n",
    "    batch_size=1,\n",
    "    num_workers=0)\n",
    "\n",
    "def predict(model, dataloader):\n",
    "    prediction_list = []\n",
    "    for i, batch in enumerate(dataloader):\n",
    "        test  = Variable(batch.view(-1, 512, 768))\n",
    "        outputs = model(test)\n",
    "        _, predicted = torch.max(outputs.data, 1) \n",
    "        prediction_list.append(predicted.cpu())\n",
    "    return prediction_list\n",
    "\n",
    "predictions_gru = predict(model_gru, test_loader_predict)\n",
    "\n",
    "a_tensor = torch.IntTensor(predictions_gru)\n",
    "b_tensor = torch.IntTensor(test_targets)\n",
    "\n",
    "accuracy = Accuracy()\n",
    "gru_test_acc = accuracy(a_tensor, b_tensor).item()\n",
    "\n",
    "f1 = F1Score(num_classes=2, average='macro') # checked if weighted can be used\n",
    "gru_f1_score = f1(a_tensor, b_tensor).item()\n",
    "\n",
    "precision = Precision(average='macro', num_classes=2)\n",
    "gru_precision = precision(a_tensor, b_tensor).item()\n",
    "\n",
    "recall = Recall(average='macro', num_classes=2)\n",
    "gru_recall = recall(a_tensor, b_tensor).item()\n",
    "\n",
    "print(\"Accuracy of Gated RNN :\", gru_test_acc)\n",
    "print(\"F1 score of Gated RNN :\", gru_f1_score)\n",
    "print(\"Precision of Gated RNN :\", gru_precision)\n",
    "print(\"Recall of Gated RNN :\", gru_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "32490444",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+----------------+---------------+----------+-----------+---------+\n",
      "|   Model   | Train accuracy | Test Accuracy | F1-score | Precision | Recall  |\n",
      "+===========+================+===============+==========+===========+=========+\n",
      "| Gated RNN | 0.73113        | 0.75465       | 0.57048  | 0.56694   | 0.62380 |\n",
      "+-----------+----------------+---------------+----------+-----------+---------+ \n",
      "\n"
     ]
    }
   ],
   "source": [
    "table = Texttable()\n",
    "table.set_cols_dtype(['a','f','f','f','f','f'])\n",
    "table.set_precision(5)\n",
    "table.add_rows([[\"Model\", \"Train accuracy\", \"Test Accuracy\", \"F1-score\", \"Precision\", \"Recall\"],\n",
    "                [\"Gated RNN\", gru_train_acc, gru_test_acc, gru_f1_score, gru_precision, gru_recall]])\n",
    "print(table.draw(),\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3cdd129",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"5\">LSTM</font><br>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "95d8d9cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LSTM\n",
    "\n",
    "class LSTM_Network(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        \n",
    "        super(LSTM_Network, self).__init__()\n",
    "\n",
    "         # Number of hidden dimensions\n",
    "        self.hidden_dim = hidden_dim\n",
    "        \n",
    "        # RNN\n",
    "        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=1, batch_first=True)\n",
    "        \n",
    "        # Readout layer\n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        # Initialize hidden state with zeros\n",
    "        h0 = Variable(torch.zeros(1, x.size(0), self.hidden_dim))\n",
    "        c0 = Variable(torch.zeros(1, x.size(0), self.hidden_dim))\n",
    "        \n",
    "        # One time step\n",
    "        out, (hn, cn) = self.lstm(x, (h0, c0))\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e84b0dc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_LSTM(model, train_loader, optimizer, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in train_loader:\n",
    "        \n",
    "        train  = Variable(batch[0].view(-1, 512, 768))\n",
    "        labels = Variable(batch[1])\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "                \n",
    "        output = model(train)\n",
    "        \n",
    "        loss = criterion(output, labels) \n",
    "        \n",
    "        acc = binary_accuracy(output.argmax(-1), labels)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(train_loader), epoch_acc / len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "05618834",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM_Network(\n",
      "  (lstm): LSTM(768, 20, batch_first=True)\n",
      "  (fc): Linear(in_features=20, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model_lstm = LSTM_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)\n",
    "print(model_lstm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e6f054f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 31s\n",
      "\tTrain Loss:     0.687 | Train Acc: 71.66%\n",
      "Epoch: 02 | Epoch Time: 0m 29s\n",
      "\tTrain Loss:     0.667 | Train Acc: 77.96%\n",
      "Epoch: 03 | Epoch Time: 0m 28s\n",
      "\tTrain Loss:     0.659 | Train Acc: 72.32%\n",
      "Epoch: 04 | Epoch Time: 0m 27s\n",
      "\tTrain Loss:     0.649 | Train Acc: 70.26%\n",
      "Epoch: 05 | Epoch Time: 0m 28s\n",
      "\tTrain Loss:     0.639 | Train Acc: 68.76%\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(weight = torch.FloatTensor([1/train_fair, 1/train_unfair]))\n",
    "\n",
    "optimizer = optim.Adam(model_lstm.parameters(), lr=1e-4)\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    lstm_train_loss, lstm_train_acc = train_LSTM(model_lstm,\n",
    "                                      train_loader,\n",
    "                                      optimizer,\n",
    "                                      criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: \\\n",
    "    {lstm_train_loss:.3f} | Train Acc: {lstm_train_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "eb28b03f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of LSTM : 0.6909187436103821\n",
      "F1 score of LSTM : 0.5431151390075684\n",
      "Precision of LSTM : 0.5592541098594666\n",
      "Recall of of LSTM : 0.6327338218688965\n"
     ]
    }
   ],
   "source": [
    "test_loader_predict = torch.utils.data.DataLoader(\n",
    "    test_data,\n",
    "    batch_size=1,\n",
    "    num_workers=num_workers)\n",
    "\n",
    "def predict(model, dataloader):\n",
    "    prediction_list = []\n",
    "    for i, batch in enumerate(dataloader):\n",
    "        test  = Variable(batch.view(-1, 512, 768))\n",
    "        outputs = model(test)\n",
    "        _, predicted = torch.max(outputs.data, 1) \n",
    "        prediction_list.append(predicted.cpu())\n",
    "    return prediction_list\n",
    "\n",
    "predictions_lstm = predict(model_lstm, test_loader_predict)\n",
    "\n",
    "a_tensor = torch.IntTensor(predictions_lstm)\n",
    "b_tensor = torch.IntTensor(test_targets)\n",
    "\n",
    "accuracy = Accuracy()\n",
    "lstm_test_acc = accuracy(a_tensor, b_tensor).item()\n",
    "\n",
    "f1 = F1Score(num_classes=2, average='macro') # checked if weighted can be used\n",
    "lstm_f1_score = f1(a_tensor, b_tensor).item()\n",
    "\n",
    "precision = Precision(average='macro', num_classes=2)\n",
    "lstm_precision = precision(a_tensor, b_tensor).item()\n",
    "\n",
    "recall = Recall(average='macro', num_classes=2)\n",
    "lstm_recall = recall(a_tensor, b_tensor).item()\n",
    "\n",
    "print(\"Accuracy of LSTM :\", lstm_test_acc)\n",
    "print(\"F1 score of LSTM :\", lstm_f1_score)\n",
    "print(\"Precision of LSTM :\", lstm_precision)\n",
    "print(\"Recall of of LSTM :\", lstm_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b5d21d51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------------+---------------+----------+-----------+---------+\n",
      "| Model | Train accuracy | Test Accuracy | F1-score | Precision | Recall  |\n",
      "+=======+================+===============+==========+===========+=========+\n",
      "| LSTM  | 0.68761        | 0.69092       | 0.54312  | 0.55925   | 0.63273 |\n",
      "+-------+----------------+---------------+----------+-----------+---------+ \n",
      "\n"
     ]
    }
   ],
   "source": [
    "table = Texttable()\n",
    "table.set_cols_dtype(['a','f','f','f','f','f'])\n",
    "table.set_precision(5)\n",
    "table.add_rows([[\"Model\", \"Train accuracy\", \"Test Accuracy\", \"F1-score\", \"Precision\", \"Recall\"],\n",
    "                [\"LSTM\", lstm_train_acc, lstm_test_acc, lstm_f1_score, lstm_precision, lstm_recall]])\n",
    "print(table.draw(),\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8cff721",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"5\">Bi-LSTM</font><br>\n",
    "</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f3bcb948",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LSTM\n",
    "\n",
    "class Bi_LSTM_Network(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        \n",
    "        super(Bi_LSTM_Network, self).__init__()\n",
    "\n",
    "         # Number of hidden dimensions\n",
    "        self.hidden_dim = hidden_dim\n",
    "        \n",
    "        # RNN\n",
    "        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=1, batch_first=True, bidirectional=True)\n",
    "        \n",
    "        # Readout layer\n",
    "        self.fc = nn.Linear(hidden_dim*2, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        # Initialize hidden state with zeros\n",
    "        h0 = Variable(torch.zeros(1 * 2, x.size(0), self.hidden_dim))\n",
    "        c0 = Variable(torch.zeros(1 * 2, x.size(0), self.hidden_dim))\n",
    "        \n",
    "        # One time step\n",
    "        out, (hn, cn) = self.lstm(x, (h0, c0))\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7a4693f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_Bi_LSTM(model, train_loader, optimizer, criterion):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in train_loader:\n",
    "        \n",
    "        train  = Variable(batch[0].view(-1, 512, 768))\n",
    "        labels = Variable(batch[1])\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "                \n",
    "        output = model(train)\n",
    "        \n",
    "        loss = criterion(output, labels) \n",
    "        \n",
    "        acc = binary_accuracy(output.argmax(-1), labels)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(train_loader), epoch_acc / len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "24a277bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bi_LSTM_Network(\n",
      "  (lstm): LSTM(768, 20, batch_first=True, bidirectional=True)\n",
      "  (fc): Linear(in_features=40, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model_bi_lstm = Bi_LSTM_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)\n",
    "print(model_bi_lstm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b75537cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 50s\n",
      "\tTrain Loss:     0.687 | Train Acc: 83.90%\n",
      "Epoch: 02 | Epoch Time: 0m 47s\n",
      "\tTrain Loss:     0.673 | Train Acc: 81.49%\n",
      "Epoch: 03 | Epoch Time: 0m 49s\n",
      "\tTrain Loss:     0.652 | Train Acc: 72.41%\n",
      "Epoch: 04 | Epoch Time: 0m 48s\n",
      "\tTrain Loss:     0.643 | Train Acc: 77.69%\n",
      "Epoch: 05 | Epoch Time: 0m 47s\n",
      "\tTrain Loss:     0.633 | Train Acc: 74.51%\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(weight = torch.FloatTensor([1/train_fair, 1/train_unfair]))\n",
    "\n",
    "optimizer = optim.Adam(model_bi_lstm.parameters(), lr=1e-4)\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    bi_lstm_train_loss, bi_lstm_train_acc = train_Bi_LSTM(model_bi_lstm,\n",
    "                                      train_loader,\n",
    "                                      optimizer,\n",
    "                                      criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: \\\n",
    "    {bi_lstm_train_loss:.3f} | Train Acc: {bi_lstm_train_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "44a07e8d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of LSTM : 0.6123207807540894\n",
      "F1 score of LSTM : 0.509839653968811\n",
      "Precision of LSTM : 0.561344563961029\n",
      "Recall of of LSTM : 0.6546074151992798\n"
     ]
    }
   ],
   "source": [
    "test_loader_predict = torch.utils.data.DataLoader(\n",
    "    test_data,\n",
    "    batch_size=1,\n",
    "    num_workers=num_workers)\n",
    "\n",
    "def predict(model, dataloader):\n",
    "    prediction_list = []\n",
    "    for i, batch in enumerate(dataloader):\n",
    "        test  = Variable(batch.view(-1, 512, 768))\n",
    "        outputs = model(test)\n",
    "        _, predicted = torch.max(outputs.data, 1) \n",
    "        prediction_list.append(predicted.cpu())\n",
    "    return prediction_list\n",
    "\n",
    "predictions_bi_lstm = predict(model_bi_lstm, test_loader_predict)\n",
    "\n",
    "a_tensor = torch.IntTensor(predictions_bi_lstm)\n",
    "b_tensor = torch.IntTensor(test_targets)\n",
    "\n",
    "accuracy = Accuracy()\n",
    "bi_lstm_test_acc = accuracy(a_tensor, b_tensor).item()\n",
    "\n",
    "f1 = F1Score(num_classes=2, average='macro') # checked if weighted can be used\n",
    "bi_lstm_f1_score = f1(a_tensor, b_tensor).item()\n",
    "\n",
    "precision = Precision(average='macro', num_classes=2)\n",
    "bi_lstm_precision = precision(a_tensor, b_tensor).item()\n",
    "\n",
    "recall = Recall(average='macro', num_classes=2)\n",
    "bi_lstm_recall = recall(a_tensor, b_tensor).item()\n",
    "\n",
    "print(\"Accuracy of LSTM :\", bi_lstm_test_acc)\n",
    "print(\"F1 score of LSTM :\", bi_lstm_f1_score)\n",
    "print(\"Precision of LSTM :\", bi_lstm_precision)\n",
    "print(\"Recall of of LSTM :\", bi_lstm_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e99372b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----------------+---------------+----------+-----------+---------+\n",
      "|  Model  | Train accuracy | Test Accuracy | F1-score | Precision | Recall  |\n",
      "+=========+================+===============+==========+===========+=========+\n",
      "| Bi-LSTM | 0.74506        | 0.61232       | 0.50984  | 0.56134   | 0.65461 |\n",
      "+---------+----------------+---------------+----------+-----------+---------+ \n",
      "\n"
     ]
    }
   ],
   "source": [
    "table = Texttable()\n",
    "table.set_cols_dtype(['a','f','f','f','f','f'])\n",
    "table.set_precision(5)\n",
    "table.add_rows([[\"Model\", \"Train accuracy\", \"Test Accuracy\", \"F1-score\", \"Precision\", \"Recall\"],\n",
    "                [\"Bi-LSTM\", bi_lstm_train_acc, bi_lstm_test_acc, bi_lstm_f1_score, bi_lstm_precision, bi_lstm_recall]])\n",
    "print(table.draw(),\"\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "23f916ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+----------------+---------------+----------+-----------+---------+\n",
      "|   Model    | Train accuracy | Test Accuracy | F1-score | Precision | Recall  |\n",
      "+============+================+===============+==========+===========+=========+\n",
      "| Simple RNN | 0.74408        | 0.74190       | 0.56979  | 0.56910   | 0.63581 |\n",
      "+------------+----------------+---------------+----------+-----------+---------+\n",
      "| Gated RNN  | 0.73113        | 0.75465       | 0.57048  | 0.56694   | 0.62380 |\n",
      "+------------+----------------+---------------+----------+-----------+---------+\n",
      "| LSTM       | 0.68761        | 0.69092       | 0.54312  | 0.55925   | 0.63273 |\n",
      "+------------+----------------+---------------+----------+-----------+---------+\n",
      "| Bi-LSTM    | 0.74506        | 0.61232       | 0.50984  | 0.56134   | 0.65461 |\n",
      "+------------+----------------+---------------+----------+-----------+---------+ \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# compare all\n",
    "table = Texttable()\n",
    "table.set_cols_dtype(['a','f','f','f','f','f'])\n",
    "table.set_precision(5)\n",
    "table.add_rows([[\"Model\", \"Train accuracy\", \"Test Accuracy\", \"F1-score\", \"Precision\", \"Recall\"],\n",
    "                [\"Simple RNN\", rnn_train_acc, rnn_test_acc, rnn_f1_score, rnn_precision, rnn_recall],\n",
    "                [\"Gated RNN\", gru_train_acc, gru_test_acc, gru_f1_score, gru_precision, gru_recall],\n",
    "                [\"LSTM\", lstm_train_acc, lstm_test_acc, lstm_f1_score, lstm_precision, lstm_recall],\n",
    "                [\"Bi-LSTM\", bi_lstm_train_acc, bi_lstm_test_acc, bi_lstm_f1_score, bi_lstm_precision, bi_lstm_recall]])\n",
    "print(table.draw(),\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95f7eb4d",
   "metadata": {},
   "source": [
    "<span style=\"color:darkviolet\">\n",
    "<font size=\"3\">Best Performance : Gated-RNN</font><br>\n",
    "</span>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}