fairness_classification/rnn_models/bert_rnn_models_final.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4fEIOgCEAoEB",
        "outputId": "be182f25-2128-439a-e889-97db1b5df9b3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N5-kKsPTD6FW",
        "outputId": "c249d54d-b0be-4d24-9e3f-98a7d13eafdd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/.shortcut-targets-by-id/1TX7I-_KwAgP0OHczEL8zSwOuxJHxViff/nlp_data\n"
          ]
        }
      ],
      "source": [
        "%cd /content/drive/My\\ Drive/nlp_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CC-2k92wbW_9",
        "outputId": "ec4008bb-43af-4e43-99a4-e861c47a2399"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pickle5 in /usr/local/lib/python3.7/dist-packages (0.0.12)\n"
          ]
        }
      ],
      "source": [
        "!pip3 install pickle5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "3sujPKXnEffO"
      },
      "outputs": [],
      "source": [
        "import pickle\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import DataLoader, TensorDataset\n",
        "from torchvision import datasets, transforms\n",
        "from torchvision.transforms import ToTensor\n",
        "from torch.autograd import Variable\n",
        "import torch.optim as optim\n",
        "import warnings\n",
        "import numpy as np\n",
        "import torch.nn.functional as F\n",
        "import pandas as pd\n",
        "from sklearn.metrics import classification_report\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import time\n",
        "import pickle5 as pickle\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "\n",
        "warnings.filterwarnings(\"ignore\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ed3Q_Ko-Ja0w",
        "outputId": "15cdb2cb-50ac-4e7e-cda9-6d0e4023f10d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Using CUDA - GPU\n"
          ]
        }
      ],
      "source": [
        "if torch.cuda.is_available():\n",
        "  device = torch.device(\"cuda\")\n",
        "  print(\"Using CUDA - GPU\")\n",
        "else:\n",
        "  device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "PLCm71o1Efpk"
      },
      "outputs": [],
      "source": [
        "# Read train BERT embeddings\n",
        "with open(\"./embeddings/train_bert_embeddings.pkl\", \"rb\") as f:\n",
        "    training_data = pickle.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-WoZIZqcL8So",
        "outputId": "e8e11a97-1443-4844-d95b-13fedb29e369"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(7531, torch.Size([1, 512, 768]))"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(training_data['embeddings']), training_data['embeddings'][0].shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "aqjqZ3MzGb2N"
      },
      "outputs": [],
      "source": [
        "# Read test BERT embeddings\n",
        "with open(\"./embeddings/test_bert_embeddings.pkl\", \"rb\") as f:\n",
        "    testing_data = pickle.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "SQR2NG7vGic2"
      },
      "outputs": [],
      "source": [
        "train_dataset_path = \"tos_clauses_train.csv\"\n",
        "test_dataset_path = \"tos_clauses_dev.csv\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "xPFuwkP3Gke8"
      },
      "outputs": [],
      "source": [
        "train_df = pd.read_csv(train_dataset_path, header=0)\n",
        "test_df = pd.read_csv(test_dataset_path, header=0)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "yirBAxyGGlZ8"
      },
      "outputs": [],
      "source": [
        "train_targets = train_df.label.values\n",
        "test_targets = test_df.label.values\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "011XEpWzMECC",
        "outputId": "3c54e868-c221-4458-c6c6-84feb22f7bc5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "7531"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_targets)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "yRMimo_2Gmcd"
      },
      "outputs": [],
      "source": [
        "class Dataset(object):\n",
        "    \"\"\"An abstract class representing a Dataset.\n",
        "    All other datasets should subclass it. All subclasses should\n",
        "    override ``__len__``, that provides the size of the dataset,\n",
        "    and ``__getitem__``, supporting integer indexing in range\n",
        "    from 0 to len(self) exclusive.\n",
        "    \"\"\"\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    def __len__(self):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    def __add__(self, other):\n",
        "        return ConcatDataset([self, other])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "6cmi-lMwGo7U"
      },
      "outputs": [],
      "source": [
        "class TOSDataset(Dataset):\n",
        "    def __init__(self, X, Y, transform=None):\n",
        "        self.data1 = X\n",
        "        self.data2 = Y\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data1)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        x = self.data1[index]\n",
        "        y = self.data2[index]\n",
        "\n",
        "        if self.transform is not None:\n",
        "            x = torch.tensor(x)\n",
        "\n",
        "        return torch.squeeze(x, dim=1), torch.tensor(y)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-vseoc8yGqnw",
        "outputId": "ed2cbf57-d547-40f7-b12b-7737e2f10877"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<torch.utils.data.sampler.SubsetRandomSampler object at 0x7fc9c1e78110>\n",
            "train_fair:6705\n",
            "train_unfair:826\n"
          ]
        }
      ],
      "source": [
        "test_len = len(test_df)\n",
        "train_len = len(train_df)\n",
        "X_train_tensor = TOSDataset(train_df[\"sentences\"], train_df[\"label\"])\n",
        "\n",
        "num_train = len(X_train_tensor)\n",
        "indices = list(range(num_train))\n",
        "np.random.shuffle(indices)\n",
        "train_sampler = SubsetRandomSampler(indices)\n",
        "print(train_sampler)\n",
        "train_df_by_index = train_df.loc[indices]\n",
        "train_fair = sum(train_df_by_index[\"label\"] == 0)\n",
        "train_unfair = sum(train_df_by_index[\"label\"] == 1)\n",
        "print(\"train_fair:\" + str(train_fair))\n",
        "print(\"train_unfair:\" + str(train_unfair))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "KRTl4fOWGruo"
      },
      "outputs": [],
      "source": [
        "train_data = TOSDataset(training_data[\"embeddings\"], train_targets, transform=transforms.ToTensor())\n",
        "test_data = TOSDataset(testing_data[\"embeddings\"], test_targets, transform=transforms.ToTensor())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "Ec9og6l6GtSO"
      },
      "outputs": [],
      "source": [
        "# how many samples per batch to load\n",
        "BATCH_SIZE = 20\n",
        "\n",
        "# number of subprocesses to use for data loading\n",
        "NUM_WORKERS = 0\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "TZrF2TcdGtTa"
      },
      "outputs": [],
      "source": [
        "# prepare data loaders\n",
        "train_loader = DataLoader(\n",
        "    train_data, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=NUM_WORKERS\n",
        ")\n",
        "test_loader = DataLoader(\n",
        "    test_data, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "grk4mFw5GtZe"
      },
      "outputs": [],
      "source": [
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "gfYi2SyMHFJL"
      },
      "outputs": [],
      "source": [
        "EMBEDDING_DIM = 768\n",
        "HIDDEN_DIM = 512\n",
        "OUTPUT_DIM = 2\n",
        "N_EPOCHS = 50"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l1hW9sKeHJLo",
        "outputId": "4cbadd2c-2234-45ec-dedc-01685ec3c2b6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([0.56159582, 4.55871671])"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "class_weight = compute_class_weight(\n",
        "    \"balanced\", classes=np.unique(train_df_by_index[\"label\"]), y=train_df_by_index[\"label\"]\n",
        ")\n",
        "class_weight"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GX0UWzm6SpW5"
      },
      "source": [
        "#**Simple RNN**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "rLHt0YaWGtdL"
      },
      "outputs": [],
      "source": [
        "class RNNet(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
        "\n",
        "        super(RNNet, self).__init__()\n",
        "\n",
        "        # Number of hidden dimensions\n",
        "        self.hidden_dim = hidden_dim\n",
        "\n",
        "        # RNN\n",
        "        self.rnn = nn.RNN(input_dim, hidden_dim, num_layers=3, batch_first=True, nonlinearity=\"relu\")\n",
        "\n",
        "        # Readout layer\n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        # Initialize hidden state with zeros\n",
        "        h0 = Variable(torch.zeros(3, x.size(0), self.hidden_dim)).to(device) # number of layers\n",
        "\n",
        "        # One time step\n",
        "        out, hn = self.rnn(x, h0)\n",
        "        out = self.fc(out[:, -1, :])\n",
        "        return out\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MkjDF6UXRBCK",
        "outputId": "0e698c82-93e4-4f31-aa11-9d301d86fb3e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.60      0.73      1677\n",
            "           1       0.17      0.66      0.27       206\n",
            "\n",
            "    accuracy                           0.60      1883\n",
            "   macro avg       0.55      0.63      0.50      1883\n",
            "weighted avg       0.85      0.60      0.68      1883\n",
            "\n",
            "Epoch: 01 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.033579 \\Test Loss: 0.033231\n",
            "Test loss decreased (inf --> 0.033231). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.34      0.51      1677\n",
            "           1       0.14      0.88      0.24       206\n",
            "\n",
            "    accuracy                           0.40      1883\n",
            "   macro avg       0.55      0.61      0.38      1883\n",
            "weighted avg       0.87      0.40      0.48      1883\n",
            "\n",
            "Epoch: 02 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.032284 \\Test Loss: 0.032683\n",
            "Test loss decreased (0.033231 --> 0.032683). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.87      0.89      1677\n",
            "           1       0.26      0.38      0.31       206\n",
            "\n",
            "    accuracy                           0.81      1883\n",
            "   macro avg       0.59      0.62      0.60      1883\n",
            "weighted avg       0.85      0.81      0.83      1883\n",
            "\n",
            "Epoch: 03 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.031287 \\Test Loss: 0.031527\n",
            "Test loss decreased (0.032683 --> 0.031527). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.89      0.91      1677\n",
            "           1       0.30      0.37      0.33       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.61      0.63      0.62      1883\n",
            "weighted avg       0.85      0.83      0.84      1883\n",
            "\n",
            "Epoch: 04 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030923 \\Test Loss: 0.031162\n",
            "Test loss decreased (0.031527 --> 0.031162). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.80      0.86      1677\n",
            "           1       0.24      0.51      0.33       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.59      0.66      0.60      1883\n",
            "weighted avg       0.86      0.77      0.80      1883\n",
            "\n",
            "Epoch: 05 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030612 \\Test Loss: 0.030345\n",
            "Test loss decreased (0.031162 --> 0.030345). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.79      0.86      1677\n",
            "           1       0.24      0.53      0.33       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.59      0.66      0.59      1883\n",
            "weighted avg       0.86      0.76      0.80      1883\n",
            "\n",
            "Epoch: 06 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030434 \\Test Loss: 0.029847\n",
            "Test loss decreased (0.030345 --> 0.029847). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.80      0.86      1677\n",
            "           1       0.24      0.52      0.33       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.59      0.66      0.59      1883\n",
            "weighted avg       0.86      0.77      0.80      1883\n",
            "\n",
            "Epoch: 07 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030125 \\Test Loss: 0.030504\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.70      0.80      1677\n",
            "           1       0.20      0.62      0.30       206\n",
            "\n",
            "    accuracy                           0.69      1883\n",
            "   macro avg       0.57      0.66      0.55      1883\n",
            "weighted avg       0.86      0.69      0.74      1883\n",
            "\n",
            "Epoch: 08 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 6012842.641798 \\Test Loss: 0.030760\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.82      0.87      1677\n",
            "           1       0.25      0.50      0.33       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.59      0.66      0.60      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 09 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.029754 \\Test Loss: 0.030267\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.84      0.88      1677\n",
            "           1       0.27      0.50      0.35       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.60      0.67      0.62      1883\n",
            "weighted avg       0.86      0.80      0.82      1883\n",
            "\n",
            "Epoch: 10 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.029361 \\Test Loss: 0.029752\n",
            "Test loss decreased (0.029847 --> 0.029752). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.85      0.89      1677\n",
            "           1       0.27      0.47      0.35       206\n",
            "\n",
            "    accuracy                           0.81      1883\n",
            "   macro avg       0.60      0.66      0.62      1883\n",
            "weighted avg       0.86      0.81      0.83      1883\n",
            "\n",
            "Epoch: 11 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028723 \\Test Loss: 0.029484\n",
            "Test loss decreased (0.029752 --> 0.029484). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.77      0.85      1677\n",
            "           1       0.23      0.57      0.33       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.58      0.67      0.59      1883\n",
            "weighted avg       0.86      0.75      0.79      1883\n",
            "\n",
            "Epoch: 12 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028576 \\Test Loss: 0.029845\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.78      0.85      1677\n",
            "           1       0.23      0.55      0.33       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.58      0.67      0.59      1883\n",
            "weighted avg       0.86      0.75      0.79      1883\n",
            "\n",
            "Epoch: 13 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.032918 \\Test Loss: 0.029348\n",
            "Test loss decreased (0.029484 --> 0.029348). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.68      0.79      1677\n",
            "           1       0.21      0.69      0.32       206\n",
            "\n",
            "    accuracy                           0.68      1883\n",
            "   macro avg       0.58      0.68      0.56      1883\n",
            "weighted avg       0.87      0.68      0.74      1883\n",
            "\n",
            "Epoch: 14 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028401 \\Test Loss: 0.029852\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.88      0.90      1677\n",
            "           1       0.30      0.42      0.35       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.61      0.65      0.62      1883\n",
            "weighted avg       0.86      0.83      0.84      1883\n",
            "\n",
            "Epoch: 15 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028119 \\Test Loss: 0.030341\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.76      0.84      1677\n",
            "           1       0.25      0.63      0.35       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.59      0.70      0.60      1883\n",
            "weighted avg       0.87      0.75      0.79      1883\n",
            "\n",
            "Epoch: 16 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.027601 \\Test Loss: 0.029150\n",
            "Test loss decreased (0.029348 --> 0.029150). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.76      0.84      1677\n",
            "           1       0.24      0.62      0.34       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.59      0.69      0.59      1883\n",
            "weighted avg       0.86      0.74      0.79      1883\n",
            "\n",
            "Epoch: 17 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.027596 \\Test Loss: 0.028793\n",
            "Test loss decreased (0.029150 --> 0.028793). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.82      0.87      1677\n",
            "           1       0.27      0.56      0.37       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.69      0.62      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 18 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.027096 \\Test Loss: 0.028853\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.74      0.83      1677\n",
            "           1       0.23      0.64      0.34       206\n",
            "\n",
            "    accuracy                           0.73      1883\n",
            "   macro avg       0.59      0.69      0.58      1883\n",
            "weighted avg       0.87      0.73      0.78      1883\n",
            "\n",
            "Epoch: 19 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 3.178327 \\Test Loss: 0.029448\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.83      0.88      1677\n",
            "           1       0.27      0.50      0.35       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.60      0.67      0.62      1883\n",
            "weighted avg       0.86      0.80      0.82      1883\n",
            "\n",
            "Epoch: 20 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028476 \\Test Loss: 0.029595\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.78      0.86      1677\n",
            "           1       0.26      0.63      0.37       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.60      0.70      0.61      1883\n",
            "weighted avg       0.87      0.76      0.80      1883\n",
            "\n",
            "Epoch: 21 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028027 \\Test Loss: 0.028971\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.84      0.88      1677\n",
            "           1       0.27      0.48      0.35       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.60      0.66      0.62      1883\n",
            "weighted avg       0.86      0.80      0.83      1883\n",
            "\n",
            "Epoch: 22 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.027184 \\Test Loss: 0.030396\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.65      0.77      1677\n",
            "           1       0.21      0.74      0.32       206\n",
            "\n",
            "    accuracy                           0.66      1883\n",
            "   macro avg       0.58      0.70      0.55      1883\n",
            "weighted avg       0.87      0.66      0.73      1883\n",
            "\n",
            "Epoch: 23 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.026930 \\Test Loss: 0.029279\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.78      0.85      1677\n",
            "           1       0.26      0.64      0.37       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.60      0.71      0.61      1883\n",
            "weighted avg       0.87      0.76      0.80      1883\n",
            "\n",
            "Epoch: 24 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.026734 \\Test Loss: 0.029145\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.88      0.90      1677\n",
            "           1       0.29      0.41      0.34       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.61      0.64      0.62      1883\n",
            "weighted avg       0.85      0.82      0.84      1883\n",
            "\n",
            "Epoch: 25 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.026646 \\Test Loss: 0.030319\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.82      0.87      1677\n",
            "           1       0.28      0.57      0.37       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.61      0.69      0.62      1883\n",
            "weighted avg       0.87      0.79      0.82      1883\n",
            "\n",
            "Epoch: 26 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.026758 \\Test Loss: 0.029711\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.86      0.89      1677\n",
            "           1       0.30      0.49      0.37       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.62      0.67      0.63      1883\n",
            "weighted avg       0.86      0.82      0.84      1883\n",
            "\n",
            "Epoch: 27 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.026178 \\Test Loss: 0.029638\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.74      0.83      1677\n",
            "           1       0.24      0.67      0.35       206\n",
            "\n",
            "    accuracy                           0.73      1883\n",
            "   macro avg       0.59      0.70      0.59      1883\n",
            "weighted avg       0.87      0.73      0.78      1883\n",
            "\n",
            "Epoch: 28 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.026303 \\Test Loss: 0.028496\n",
            "Test loss decreased (0.028793 --> 0.028496). Saving model...\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.92      0.92      1677\n",
            "           1       0.29      0.27      0.28       206\n",
            "\n",
            "    accuracy                           0.85      1883\n",
            "   macro avg       0.60      0.59      0.60      1883\n",
            "weighted avg       0.84      0.85      0.85      1883\n",
            "\n",
            "Epoch: 29 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 81451.261133 \\Test Loss: 0.033031\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.59      0.73      1677\n",
            "           1       0.18      0.74      0.29       206\n",
            "\n",
            "    accuracy                           0.60      1883\n",
            "   macro avg       0.56      0.66      0.51      1883\n",
            "weighted avg       0.86      0.60      0.68      1883\n",
            "\n",
            "Epoch: 30 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.031026 \\Test Loss: 0.031205\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.87      0.90      1677\n",
            "           1       0.29      0.42      0.34       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.61      0.65      0.62      1883\n",
            "weighted avg       0.86      0.82      0.84      1883\n",
            "\n",
            "Epoch: 31 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.029483 \\Test Loss: 0.030342\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.79      0.85      1677\n",
            "           1       0.24      0.56      0.34       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.59      0.67      0.60      1883\n",
            "weighted avg       0.86      0.76      0.80      1883\n",
            "\n",
            "Epoch: 32 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028753 \\Test Loss: 0.029919\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.75      0.84      1677\n",
            "           1       0.24      0.62      0.34       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.59      0.69      0.59      1883\n",
            "weighted avg       0.86      0.74      0.78      1883\n",
            "\n",
            "Epoch: 33 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028322 \\Test Loss: 0.029395\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.87      0.90      1677\n",
            "           1       0.28      0.42      0.34       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.60      0.64      0.62      1883\n",
            "weighted avg       0.85      0.82      0.84      1883\n",
            "\n",
            "Epoch: 34 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.028145 \\Test Loss: 0.030334\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.82      0.87      1677\n",
            "           1       0.26      0.52      0.35       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.67      0.61      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 35 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.027763 \\Test Loss: 0.029477\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.00      0.00      0.00      1677\n",
            "           1       0.11      1.00      0.20       206\n",
            "\n",
            "    accuracy                           0.11      1883\n",
            "   macro avg       0.05      0.50      0.10      1883\n",
            "weighted avg       0.01      0.11      0.02      1883\n",
            "\n",
            "Epoch: 36 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 14521705883204.488281 \\Test Loss: 0.087753\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.78      0.85      1677\n",
            "           1       0.21      0.48      0.29       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.57      0.63      0.57      1883\n",
            "weighted avg       0.85      0.75      0.78      1883\n",
            "\n",
            "Epoch: 37 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.105356 \\Test Loss: 0.032410\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.80      0.86      1677\n",
            "           1       0.22      0.47      0.30       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.57      0.63      0.58      1883\n",
            "weighted avg       0.85      0.77      0.80      1883\n",
            "\n",
            "Epoch: 38 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.032385 \\Test Loss: 0.031671\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.95      0.93      1677\n",
            "           1       0.34      0.22      0.27       206\n",
            "\n",
            "    accuracy                           0.87      1883\n",
            "   macro avg       0.62      0.58      0.60      1883\n",
            "weighted avg       0.85      0.87      0.86      1883\n",
            "\n",
            "Epoch: 39 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.031848 \\Test Loss: 0.033744\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.71      0.81      1677\n",
            "           1       0.21      0.62      0.31       206\n",
            "\n",
            "    accuracy                           0.70      1883\n",
            "   macro avg       0.57      0.66      0.56      1883\n",
            "weighted avg       0.86      0.70      0.75      1883\n",
            "\n",
            "Epoch: 40 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.031553 \\Test Loss: 0.031190\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.91      0.91      1677\n",
            "           1       0.30      0.32      0.31       206\n",
            "\n",
            "    accuracy                           0.84      1883\n",
            "   macro avg       0.61      0.61      0.61      1883\n",
            "weighted avg       0.85      0.84      0.85      1883\n",
            "\n",
            "Epoch: 41 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.031133 \\Test Loss: 0.031901\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.76      0.84      1677\n",
            "           1       0.24      0.60      0.34       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.59      0.68      0.59      1883\n",
            "weighted avg       0.86      0.74      0.79      1883\n",
            "\n",
            "Epoch: 42 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030762 \\Test Loss: 0.030489\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.77      0.84      1677\n",
            "           1       0.24      0.60      0.34       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.59      0.68      0.59      1883\n",
            "weighted avg       0.86      0.75      0.79      1883\n",
            "\n",
            "Epoch: 43 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030344 \\Test Loss: 0.030217\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.68      0.79      1677\n",
            "           1       0.20      0.65      0.31       206\n",
            "\n",
            "    accuracy                           0.68      1883\n",
            "   macro avg       0.57      0.66      0.55      1883\n",
            "weighted avg       0.86      0.68      0.74      1883\n",
            "\n",
            "Epoch: 44 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030089 \\Test Loss: 0.030455\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.81      0.87      1677\n",
            "           1       0.26      0.52      0.34       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.59      0.67      0.61      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 45 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.029852 \\Test Loss: 0.030131\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.66      0.78      1677\n",
            "           1       0.20      0.67      0.30       206\n",
            "\n",
            "    accuracy                           0.66      1883\n",
            "   macro avg       0.57      0.67      0.54      1883\n",
            "weighted avg       0.86      0.66      0.73      1883\n",
            "\n",
            "Epoch: 46 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.029876 \\Test Loss: 0.030570\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.88      0.90      1677\n",
            "           1       0.29      0.40      0.33       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.60      0.64      0.62      1883\n",
            "weighted avg       0.85      0.83      0.84      1883\n",
            "\n",
            "Epoch: 47 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.029778 \\Test Loss: 0.030853\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.64      0.77      1677\n",
            "           1       0.19      0.70      0.30       206\n",
            "\n",
            "    accuracy                           0.65      1883\n",
            "   macro avg       0.57      0.67      0.53      1883\n",
            "weighted avg       0.86      0.65      0.71      1883\n",
            "\n",
            "Epoch: 48 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 0.030134 \\Test Loss: 0.030482\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.89      0.91      1677\n",
            "           1       0.30      0.38      0.34       206\n",
            "\n",
            "    accuracy                           0.84      1883\n",
            "   macro avg       0.61      0.64      0.62      1883\n",
            "weighted avg       0.85      0.84      0.84      1883\n",
            "\n",
            "Epoch: 49 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 33040889435950712096699711488.000000 \\Test Loss: 0.030968\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.83      0.88      1677\n",
            "           1       0.26      0.49      0.34       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.66      0.61      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 50 | Epoch Time: 0m 24s\n",
            "\tTraining Loss: 41321357409646898018709929984.000000 \\Test Loss: 0.029863\n",
            "======================================================================\n"
          ]
        }
      ],
      "source": [
        "model = RNNet(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weight)).to(device)\n",
        "optimizer = optim.Adam(model.parameters(), lr=1e-4)\n",
        "test_min_loss = np.inf\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "\n",
        "    start_time = time.time()\n",
        "    model.train()\n",
        "    train_loss = 0.0\n",
        "    test_loss = 0.0\n",
        "    for inputs, target in train_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        model.zero_grad()\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model(inputs)\n",
        "        loss = loss_fn(output, target)\n",
        "        loss.backward()\n",
        "        train_loss += loss.item()\n",
        "        optimizer.step()\n",
        "\n",
        "    y_pred_list = []\n",
        "    y_targ_list = []\n",
        "    model.eval()\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model(inputs)\n",
        "        loss = loss_fn(output, target)\n",
        "        test_loss += loss.item()\n",
        "        _, y_test_pred = torch.max(output, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "    \n",
        "\n",
        "    if(epoch%1 == 0):\n",
        "        y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "        y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "        y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "        y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "        print(classification_report(y_targ_list, y_pred_list))\n",
        "\n",
        "    train_loss = train_loss / len(train_loader.dataset)\n",
        "    test_loss = test_loss / len(test_loader.dataset)\n",
        "\n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "\n",
        "    print(f\"Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s\")\n",
        "    print(\"\\tTraining Loss: {:.6f} \\Test Loss: {:.6f}\".format(train_loss, test_loss))\n",
        "    if test_loss <= test_min_loss:\n",
        "        print(\"Test loss decreased ({:.6f} --> {:.6f}). Saving model...\".format(test_min_loss, test_loss))\n",
        "        torch.save(model.state_dict(), \"models/rnn_bert_model.pt\")\n",
        "        test_min_loss = test_loss\n",
        "    print(\"=\"*70)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yWKGcX34QcFr",
        "outputId": "981c4dfd-2778-4361-fcd8-df4c0b93d418"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.74      0.83      1677\n",
            "           1       0.24      0.67      0.35       206\n",
            "\n",
            "    accuracy                           0.73      1883\n",
            "   macro avg       0.59      0.70      0.59      1883\n",
            "weighted avg       0.87      0.73      0.78      1883\n",
            "\n"
          ]
        }
      ],
      "source": [
        "y_pred_list = []\n",
        "y_targ_list = []\n",
        "model = RNNet(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "model.load_state_dict(torch.load(\"models/rnn_bert_model.pt\"))\n",
        "model.eval()\n",
        "\n",
        "with torch.no_grad():\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        y_test_pred = model(inputs)\n",
        "        _, y_test_pred = torch.max(y_test_pred, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "\n",
        "y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "print(classification_report(y_targ_list, y_pred_list))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N1ckBuHiS0uk"
      },
      "source": [
        "#**Gated RNN**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "eg3iX7GaS5HO"
      },
      "outputs": [],
      "source": [
        "class GRU_Network(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
        "\n",
        "        super(GRU_Network, self).__init__()\n",
        "\n",
        "        # Number of hidden dimensions\n",
        "        self.hidden_dim = hidden_dim\n",
        "\n",
        "        # RNN\n",
        "        self.rnn = nn.GRU(input_dim, hidden_dim, num_layers=1, batch_first=True)\n",
        "\n",
        "        # Readout layer\n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        # Initialize hidden state with zeros\n",
        "        h0 = Variable(torch.zeros(1, x.size(0), self.hidden_dim)).to(device)\n",
        "\n",
        "        # One time step\n",
        "        out, hn = self.rnn(x, h0)\n",
        "        out = self.fc(out[:, -1, :])\n",
        "        return out\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mdFuCo-jS5Ql",
        "outputId": "70c7ff34-8917-40ae-e5d1-66de7bdca4a8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.85      0.89      1677\n",
            "           1       0.32      0.58      0.41       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.63      0.71      0.65      1883\n",
            "weighted avg       0.87      0.82      0.84      1883\n",
            "\n",
            "Epoch: 01 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.034934 \\Test Loss: 0.029151\n",
            "Test loss decreased (inf --> 0.029151). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.97      0.94      1677\n",
            "           1       0.57      0.31      0.40       206\n",
            "\n",
            "    accuracy                           0.90      1883\n",
            "   macro avg       0.74      0.64      0.67      1883\n",
            "weighted avg       0.88      0.90      0.88      1883\n",
            "\n",
            "Epoch: 02 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028747 \\Test Loss: 0.031566\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.91      0.93      1677\n",
            "           1       0.44      0.59      0.51       206\n",
            "\n",
            "    accuracy                           0.87      1883\n",
            "   macro avg       0.70      0.75      0.72      1883\n",
            "weighted avg       0.89      0.87      0.88      1883\n",
            "\n",
            "Epoch: 03 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025157 \\Test Loss: 0.025749\n",
            "Test loss decreased (0.029151 --> 0.025749). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.89      0.92      1677\n",
            "           1       0.44      0.68      0.53       206\n",
            "\n",
            "    accuracy                           0.87      1883\n",
            "   macro avg       0.70      0.79      0.73      1883\n",
            "weighted avg       0.90      0.87      0.88      1883\n",
            "\n",
            "Epoch: 04 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.023271 \\Test Loss: 0.023849\n",
            "Test loss decreased (0.025749 --> 0.023849). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.63      0.76      1677\n",
            "           1       0.23      0.89      0.36       206\n",
            "\n",
            "    accuracy                           0.66      1883\n",
            "   macro avg       0.60      0.76      0.56      1883\n",
            "weighted avg       0.90      0.66      0.72      1883\n",
            "\n",
            "Epoch: 05 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.021704 \\Test Loss: 0.026288\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.68      0.80      1677\n",
            "           1       0.25      0.86      0.38       206\n",
            "\n",
            "    accuracy                           0.70      1883\n",
            "   macro avg       0.61      0.77      0.59      1883\n",
            "weighted avg       0.90      0.70      0.75      1883\n",
            "\n",
            "Epoch: 06 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.019851 \\Test Loss: 0.025508\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.94      0.94      1677\n",
            "           1       0.54      0.52      0.53       206\n",
            "\n",
            "    accuracy                           0.90      1883\n",
            "   macro avg       0.74      0.73      0.73      1883\n",
            "weighted avg       0.90      0.90      0.90      1883\n",
            "\n",
            "Epoch: 07 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.019627 \\Test Loss: 0.027068\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.97      0.95      1677\n",
            "           1       0.64      0.47      0.54       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.79      0.72      0.75      1883\n",
            "weighted avg       0.90      0.91      0.91      1883\n",
            "\n",
            "Epoch: 08 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.017437 \\Test Loss: 0.032468\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.91      0.93      1677\n",
            "           1       0.47      0.65      0.54       206\n",
            "\n",
            "    accuracy                           0.88      1883\n",
            "   macro avg       0.71      0.78      0.74      1883\n",
            "weighted avg       0.90      0.88      0.89      1883\n",
            "\n",
            "Epoch: 09 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.015967 \\Test Loss: 0.024433\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.97      0.84      0.90      1677\n",
            "           1       0.37      0.76      0.50       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.67      0.80      0.70      1883\n",
            "weighted avg       0.90      0.83      0.86      1883\n",
            "\n",
            "Epoch: 10 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.015065 \\Test Loss: 0.021829\n",
            "Test loss decreased (0.023849 --> 0.021829). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.80      0.87      1677\n",
            "           1       0.32      0.76      0.45       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.64      0.78      0.66      1883\n",
            "weighted avg       0.89      0.80      0.83      1883\n",
            "\n",
            "Epoch: 11 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.014873 \\Test Loss: 0.024149\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.94      0.94      1677\n",
            "           1       0.53      0.60      0.56       206\n",
            "\n",
            "    accuracy                           0.90      1883\n",
            "   macro avg       0.74      0.77      0.75      1883\n",
            "weighted avg       0.90      0.90      0.90      1883\n",
            "\n",
            "Epoch: 12 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.012728 \\Test Loss: 0.027109\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.95      0.95      1677\n",
            "           1       0.59      0.56      0.57       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.77      0.76      0.76      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 13 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.011868 \\Test Loss: 0.029051\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.96      0.95      1677\n",
            "           1       0.62      0.57      0.59       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.78      0.76      0.77      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 14 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.011019 \\Test Loss: 0.031355\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.90      0.93      1677\n",
            "           1       0.47      0.69      0.56       206\n",
            "\n",
            "    accuracy                           0.88      1883\n",
            "   macro avg       0.71      0.80      0.75      1883\n",
            "weighted avg       0.91      0.88      0.89      1883\n",
            "\n",
            "Epoch: 15 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.011046 \\Test Loss: 0.024273\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.97      0.95      1677\n",
            "           1       0.65      0.48      0.55       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.79      0.72      0.75      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 16 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.009578 \\Test Loss: 0.036679\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.92      0.94      1677\n",
            "           1       0.50      0.63      0.56       206\n",
            "\n",
            "    accuracy                           0.89      1883\n",
            "   macro avg       0.73      0.77      0.75      1883\n",
            "weighted avg       0.90      0.89      0.90      1883\n",
            "\n",
            "Epoch: 17 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.008347 \\Test Loss: 0.025960\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.87      0.91      1677\n",
            "           1       0.41      0.72      0.52       206\n",
            "\n",
            "    accuracy                           0.86      1883\n",
            "   macro avg       0.69      0.80      0.72      1883\n",
            "weighted avg       0.90      0.86      0.87      1883\n",
            "\n",
            "Epoch: 18 | Epoch Time: 0m 14s\n",
            "\tTraining Loss: 0.008835 \\Test Loss: 0.024223\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.97      0.96      1677\n",
            "           1       0.66      0.54      0.59       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.80      0.75      0.77      1883\n",
            "weighted avg       0.91      0.92      0.92      1883\n",
            "\n",
            "Epoch: 19 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.007554 \\Test Loss: 0.036552\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.97      0.96      1677\n",
            "           1       0.69      0.54      0.61       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.82      0.75      0.78      1883\n",
            "weighted avg       0.92      0.92      0.92      1883\n",
            "\n",
            "Epoch: 20 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.006466 \\Test Loss: 0.042559\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.97      0.96      1677\n",
            "           1       0.66      0.54      0.60       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.80      0.75      0.78      1883\n",
            "weighted avg       0.91      0.92      0.92      1883\n",
            "\n",
            "Epoch: 21 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.005764 \\Test Loss: 0.037656\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.92      0.94      1677\n",
            "           1       0.52      0.65      0.58       206\n",
            "\n",
            "    accuracy                           0.89      1883\n",
            "   macro avg       0.74      0.79      0.76      1883\n",
            "weighted avg       0.91      0.89      0.90      1883\n",
            "\n",
            "Epoch: 22 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.005613 \\Test Loss: 0.030052\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.97      0.96      1677\n",
            "           1       0.67      0.53      0.59       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.81      0.75      0.77      1883\n",
            "weighted avg       0.91      0.92      0.92      1883\n",
            "\n",
            "Epoch: 23 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.005425 \\Test Loss: 0.040633\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.95      0.95      1677\n",
            "           1       0.58      0.61      0.60       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.77      0.78      0.77      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 24 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.004700 \\Test Loss: 0.035755\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.94      0.95      1677\n",
            "           1       0.57      0.62      0.59       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.76      0.78      0.77      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 25 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.004053 \\Test Loss: 0.037842\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.92      0.94      1677\n",
            "           1       0.52      0.66      0.58       206\n",
            "\n",
            "    accuracy                           0.90      1883\n",
            "   macro avg       0.74      0.79      0.76      1883\n",
            "weighted avg       0.91      0.90      0.90      1883\n",
            "\n",
            "Epoch: 26 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.003653 \\Test Loss: 0.032822\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.90      0.93      1677\n",
            "           1       0.46      0.71      0.56       206\n",
            "\n",
            "    accuracy                           0.88      1883\n",
            "   macro avg       0.71      0.80      0.75      1883\n",
            "weighted avg       0.91      0.88      0.89      1883\n",
            "\n",
            "Epoch: 27 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.004971 \\Test Loss: 0.033189\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.97      0.96      1677\n",
            "           1       0.69      0.56      0.62       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.82      0.76      0.79      1883\n",
            "weighted avg       0.92      0.92      0.92      1883\n",
            "\n",
            "Epoch: 28 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.003911 \\Test Loss: 0.048857\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.98      0.96      1677\n",
            "           1       0.75      0.50      0.60       206\n",
            "\n",
            "    accuracy                           0.93      1883\n",
            "   macro avg       0.84      0.74      0.78      1883\n",
            "weighted avg       0.92      0.93      0.92      1883\n",
            "\n",
            "Epoch: 29 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.004412 \\Test Loss: 0.055473\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.96      0.95      1677\n",
            "           1       0.64      0.58      0.61       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.80      0.77      0.78      1883\n",
            "weighted avg       0.92      0.92      0.92      1883\n",
            "\n",
            "Epoch: 30 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.003336 \\Test Loss: 0.044326\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.97      0.96      1677\n",
            "           1       0.71      0.53      0.61       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.82      0.75      0.78      1883\n",
            "weighted avg       0.92      0.92      0.92      1883\n",
            "\n",
            "Epoch: 31 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.002184 \\Test Loss: 0.058518\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.96      0.96      1677\n",
            "           1       0.65      0.61      0.63       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.80      0.78      0.79      1883\n",
            "weighted avg       0.92      0.92      0.92      1883\n",
            "\n",
            "Epoch: 32 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.002839 \\Test Loss: 0.046731\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.98      0.96      1677\n",
            "           1       0.73      0.53      0.62       206\n",
            "\n",
            "    accuracy                           0.93      1883\n",
            "   macro avg       0.84      0.76      0.79      1883\n",
            "weighted avg       0.92      0.93      0.92      1883\n",
            "\n",
            "Epoch: 33 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.002216 \\Test Loss: 0.060033\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.95      0.95      1677\n",
            "           1       0.59      0.62      0.60       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.77      0.78      0.78      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 34 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.003048 \\Test Loss: 0.043074\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.94      0.94      1677\n",
            "           1       0.55      0.63      0.59       206\n",
            "\n",
            "    accuracy                           0.90      1883\n",
            "   macro avg       0.75      0.78      0.77      1883\n",
            "weighted avg       0.91      0.90      0.91      1883\n",
            "\n",
            "Epoch: 35 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.002948 \\Test Loss: 0.045209\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.93      0.94      1677\n",
            "           1       0.53      0.67      0.59       206\n",
            "\n",
            "    accuracy                           0.90      1883\n",
            "   macro avg       0.75      0.80      0.77      1883\n",
            "weighted avg       0.91      0.90      0.90      1883\n",
            "\n",
            "Epoch: 36 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.001009 \\Test Loss: 0.042690\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.86      0.91      1677\n",
            "           1       0.40      0.74      0.52       206\n",
            "\n",
            "    accuracy                           0.85      1883\n",
            "   macro avg       0.68      0.80      0.71      1883\n",
            "weighted avg       0.90      0.85      0.87      1883\n",
            "\n",
            "Epoch: 37 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.004647 \\Test Loss: 0.032636\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.92      0.94      1677\n",
            "           1       0.49      0.66      0.56       206\n",
            "\n",
            "    accuracy                           0.89      1883\n",
            "   macro avg       0.72      0.79      0.75      1883\n",
            "weighted avg       0.91      0.89      0.89      1883\n",
            "\n",
            "Epoch: 38 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.004873 \\Test Loss: 0.038597\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.96      0.95      1677\n",
            "           1       0.62      0.58      0.60       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.78      0.77      0.78      1883\n",
            "weighted avg       0.91      0.92      0.91      1883\n",
            "\n",
            "Epoch: 39 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.002952 \\Test Loss: 0.049735\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.94      0.95      1677\n",
            "           1       0.57      0.65      0.60       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.76      0.79      0.78      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 40 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.001468 \\Test Loss: 0.049340\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.97      0.96      1677\n",
            "           1       0.69      0.52      0.59       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.81      0.75      0.77      1883\n",
            "weighted avg       0.91      0.92      0.92      1883\n",
            "\n",
            "Epoch: 41 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.003317 \\Test Loss: 0.055547\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.98      0.96      1677\n",
            "           1       0.74      0.55      0.63       206\n",
            "\n",
            "    accuracy                           0.93      1883\n",
            "   macro avg       0.84      0.76      0.80      1883\n",
            "weighted avg       0.92      0.93      0.93      1883\n",
            "\n",
            "Epoch: 42 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.001414 \\Test Loss: 0.062934\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.94      0.95      1677\n",
            "           1       0.56      0.65      0.60       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.76      0.79      0.77      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 43 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.001592 \\Test Loss: 0.045787\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.98      0.96      1677\n",
            "           1       0.75      0.55      0.64       206\n",
            "\n",
            "    accuracy                           0.93      1883\n",
            "   macro avg       0.85      0.77      0.80      1883\n",
            "weighted avg       0.92      0.93      0.93      1883\n",
            "\n",
            "Epoch: 44 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.001326 \\Test Loss: 0.065344\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.93      0.94      1677\n",
            "           1       0.52      0.63      0.57       206\n",
            "\n",
            "    accuracy                           0.90      1883\n",
            "   macro avg       0.74      0.78      0.75      1883\n",
            "weighted avg       0.91      0.90      0.90      1883\n",
            "\n",
            "Epoch: 45 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.005454 \\Test Loss: 0.043742\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.96      0.95      1677\n",
            "           1       0.63      0.58      0.60       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.79      0.77      0.78      1883\n",
            "weighted avg       0.91      0.92      0.91      1883\n",
            "\n",
            "Epoch: 46 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.003513 \\Test Loss: 0.050840\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.92      0.94      1677\n",
            "           1       0.51      0.69      0.58       206\n",
            "\n",
            "    accuracy                           0.89      1883\n",
            "   macro avg       0.73      0.80      0.76      1883\n",
            "weighted avg       0.91      0.89      0.90      1883\n",
            "\n",
            "Epoch: 47 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.001544 \\Test Loss: 0.039451\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.96      0.96      1677\n",
            "           1       0.64      0.60      0.62       206\n",
            "\n",
            "    accuracy                           0.92      1883\n",
            "   macro avg       0.80      0.78      0.79      1883\n",
            "weighted avg       0.92      0.92      0.92      1883\n",
            "\n",
            "Epoch: 48 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.000906 \\Test Loss: 0.056058\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.95      0.95      1677\n",
            "           1       0.60      0.55      0.57       206\n",
            "\n",
            "    accuracy                           0.91      1883\n",
            "   macro avg       0.77      0.75      0.76      1883\n",
            "weighted avg       0.91      0.91      0.91      1883\n",
            "\n",
            "Epoch: 49 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.003544 \\Test Loss: 0.048105\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.93      0.94      1677\n",
            "           1       0.52      0.63      0.57       206\n",
            "\n",
            "    accuracy                           0.89      1883\n",
            "   macro avg       0.73      0.78      0.75      1883\n",
            "weighted avg       0.91      0.89      0.90      1883\n",
            "\n",
            "Epoch: 50 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.001997 \\Test Loss: 0.046881\n",
            "======================================================================\n"
          ]
        }
      ],
      "source": [
        "model_gru = GRU_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "test_min_loss = np.inf\n",
        "criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weight)).to(device)\n",
        "optimizer_gru = optim.Adam(model_gru.parameters(), lr=1e-3)\n",
        "    \n",
        "for epoch in range(N_EPOCHS):\n",
        "    start_time = time.time()\n",
        "    model_gru.train()\n",
        "    train_loss = 0.0\n",
        "    test_loss = 0.0\n",
        "    for inputs, target in train_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        model_gru.zero_grad()\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model_gru(inputs)\n",
        "        loss = criterion(output, target)\n",
        "        loss.backward()\n",
        "        train_loss += loss.item()\n",
        "        optimizer_gru.step()\n",
        "\n",
        "    y_pred_list = []\n",
        "    y_targ_list = []\n",
        "    model_gru.eval()\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model_gru(inputs)\n",
        "        loss = criterion(output, target)\n",
        "        test_loss += loss.item()\n",
        "        _, y_test_pred = torch.max(output, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "    \n",
        "\n",
        "    if(epoch%1 == 0):\n",
        "        y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "        y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "        y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "        y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "        print(classification_report(y_targ_list, y_pred_list))\n",
        "\n",
        "    train_loss = train_loss / len(train_loader.dataset)\n",
        "    test_loss = test_loss / len(test_loader.dataset)\n",
        "\n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "\n",
        "    print(f\"Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s\")\n",
        "    print(\"\\tTraining Loss: {:.6f} \\Test Loss: {:.6f}\".format(train_loss, test_loss))\n",
        "    if test_loss <= test_min_loss:\n",
        "        print(\"Test loss decreased ({:.6f} --> {:.6f}). Saving model...\\n\".format(test_min_loss, test_loss))\n",
        "        torch.save(model_gru.state_dict(), \"./models/gru_bert_model.pt\")\n",
        "        test_min_loss = test_loss\n",
        "    print(\"=\"*70)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fmnbgcWAQTax",
        "outputId": "884c54c2-02f5-469b-93f8-82337bfe1a11"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.97      0.84      0.90      1677\n",
            "           1       0.37      0.76      0.50       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.67      0.80      0.70      1883\n",
            "weighted avg       0.90      0.83      0.86      1883\n",
            "\n"
          ]
        }
      ],
      "source": [
        "y_pred_list = []\n",
        "y_targ_list = []\n",
        "model = GRU_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "model.load_state_dict(torch.load(\"models/gru_bert_model.pt\"))\n",
        "model.eval()\n",
        "\n",
        "with torch.no_grad():\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        y_test_pred = model(inputs)\n",
        "        _, y_test_pred = torch.max(y_test_pred, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "\n",
        "y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "print(classification_report(y_targ_list, y_pred_list))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uZahRNS3V92S"
      },
      "source": [
        "#**LSTM**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "SmBUz1grTAlw"
      },
      "outputs": [],
      "source": [
        "# LSTM\n",
        "\n",
        "class LSTM_Network(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
        "        \n",
        "        super(LSTM_Network, self).__init__()\n",
        "\n",
        "         # Number of hidden dimensions\n",
        "        self.hidden_dim = hidden_dim\n",
        "        \n",
        "        # RNN\n",
        "        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=1, batch_first=True)\n",
        "        \n",
        "        # Readout layer\n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "\n",
        "    def forward(self, x):\n",
        "        \n",
        "        # Initialize hidden state with zeros\n",
        "        h0 = Variable(torch.zeros(1, x.size(0), self.hidden_dim)).to(device)\n",
        "        c0 = Variable(torch.zeros(1, x.size(0), self.hidden_dim)).to(device)\n",
        "        \n",
        "        # One time step\n",
        "        out, (hn, cn) = self.lstm(x, (h0, c0))\n",
        "        out = self.fc(out[:, -1, :])\n",
        "        \n",
        "        return out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pPSUf907TAov",
        "outputId": "e03c825d-8ba7-444c-c233-7181db31145b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.86      0.89      1677\n",
            "           1       0.24      0.35      0.28       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.58      0.61      0.58      1883\n",
            "weighted avg       0.84      0.80      0.82      1883\n",
            "\n",
            "Epoch: 01 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.033822 | Test Loss: 0.032260\n",
            "Test loss decreased (inf --> 0.032260). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.94      0.93      1677\n",
            "           1       0.34      0.24      0.28       206\n",
            "\n",
            "    accuracy                           0.87      1883\n",
            "   macro avg       0.62      0.59      0.60      1883\n",
            "weighted avg       0.85      0.87      0.86      1883\n",
            "\n",
            "Epoch: 02 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.032458 | Test Loss: 0.032388\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.77      0.84      1677\n",
            "           1       0.21      0.50      0.30       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.57      0.63      0.57      1883\n",
            "weighted avg       0.85      0.74      0.78      1883\n",
            "\n",
            "Epoch: 03 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.031718 | Test Loss: 0.031069\n",
            "Test loss decreased (0.032260 --> 0.031069). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.77      0.84      1677\n",
            "           1       0.22      0.53      0.31       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.58      0.65      0.58      1883\n",
            "weighted avg       0.85      0.74      0.78      1883\n",
            "\n",
            "Epoch: 04 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.031127 | Test Loss: 0.030949\n",
            "Test loss decreased (0.031069 --> 0.030949). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.80      0.86      1677\n",
            "           1       0.24      0.50      0.32       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.58      0.65      0.59      1883\n",
            "weighted avg       0.85      0.77      0.80      1883\n",
            "\n",
            "Epoch: 05 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.030579 | Test Loss: 0.030541\n",
            "Test loss decreased (0.030949 --> 0.030541). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.81      0.86      1677\n",
            "           1       0.24      0.51      0.33       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.59      0.66      0.60      1883\n",
            "weighted avg       0.86      0.77      0.81      1883\n",
            "\n",
            "Epoch: 06 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.030867 | Test Loss: 0.030525\n",
            "Test loss decreased (0.030541 --> 0.030525). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.78      0.85      1677\n",
            "           1       0.23      0.55      0.33       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.58      0.66      0.59      1883\n",
            "weighted avg       0.86      0.75      0.79      1883\n",
            "\n",
            "Epoch: 07 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.030378 | Test Loss: 0.030131\n",
            "Test loss decreased (0.030525 --> 0.030131). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.87      0.89      1677\n",
            "           1       0.28      0.41      0.33       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.60      0.64      0.61      1883\n",
            "weighted avg       0.85      0.82      0.83      1883\n",
            "\n",
            "Epoch: 08 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.030277 | Test Loss: 0.030723\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.59      0.72      1677\n",
            "           1       0.18      0.74      0.29       206\n",
            "\n",
            "    accuracy                           0.60      1883\n",
            "   macro avg       0.56      0.66      0.51      1883\n",
            "weighted avg       0.86      0.60      0.68      1883\n",
            "\n",
            "Epoch: 09 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.030018 | Test Loss: 0.031361\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.91      0.91      1677\n",
            "           1       0.31      0.34      0.33       206\n",
            "\n",
            "    accuracy                           0.84      1883\n",
            "   macro avg       0.61      0.62      0.62      1883\n",
            "weighted avg       0.85      0.84      0.85      1883\n",
            "\n",
            "Epoch: 10 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029883 | Test Loss: 0.030830\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.86      0.89      1677\n",
            "           1       0.28      0.45      0.34       206\n",
            "\n",
            "    accuracy                           0.81      1883\n",
            "   macro avg       0.60      0.65      0.62      1883\n",
            "weighted avg       0.86      0.81      0.83      1883\n",
            "\n",
            "Epoch: 11 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029499 | Test Loss: 0.030596\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.70      0.80      1677\n",
            "           1       0.21      0.63      0.31       206\n",
            "\n",
            "    accuracy                           0.69      1883\n",
            "   macro avg       0.57      0.66      0.56      1883\n",
            "weighted avg       0.86      0.69      0.75      1883\n",
            "\n",
            "Epoch: 12 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029406 | Test Loss: 0.030086\n",
            "Test loss decreased (0.030131 --> 0.030086). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.67      0.78      1677\n",
            "           1       0.20      0.67      0.31       206\n",
            "\n",
            "    accuracy                           0.67      1883\n",
            "   macro avg       0.57      0.67      0.54      1883\n",
            "weighted avg       0.86      0.67      0.73      1883\n",
            "\n",
            "Epoch: 13 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029445 | Test Loss: 0.029992\n",
            "Test loss decreased (0.030086 --> 0.029992). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.69      0.80      1677\n",
            "           1       0.21      0.65      0.31       206\n",
            "\n",
            "    accuracy                           0.69      1883\n",
            "   macro avg       0.57      0.67      0.55      1883\n",
            "weighted avg       0.86      0.69      0.74      1883\n",
            "\n",
            "Epoch: 14 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029464 | Test Loss: 0.029832\n",
            "Test loss decreased (0.029992 --> 0.029832). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.73      0.82      1677\n",
            "           1       0.22      0.62      0.32       206\n",
            "\n",
            "    accuracy                           0.71      1883\n",
            "   macro avg       0.58      0.67      0.57      1883\n",
            "weighted avg       0.86      0.71      0.76      1883\n",
            "\n",
            "Epoch: 15 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029227 | Test Loss: 0.029572\n",
            "Test loss decreased (0.029832 --> 0.029572). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.88      0.90      1677\n",
            "           1       0.28      0.39      0.33       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.60      0.64      0.61      1883\n",
            "weighted avg       0.85      0.83      0.84      1883\n",
            "\n",
            "Epoch: 16 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029058 | Test Loss: 0.029950\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.82      0.87      1677\n",
            "           1       0.26      0.51      0.34       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.59      0.67      0.61      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 17 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.029075 | Test Loss: 0.029576\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.76      0.84      1677\n",
            "           1       0.23      0.60      0.34       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.59      0.68      0.59      1883\n",
            "weighted avg       0.86      0.74      0.78      1883\n",
            "\n",
            "Epoch: 18 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028778 | Test Loss: 0.029338\n",
            "Test loss decreased (0.029572 --> 0.029338). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.78      0.85      1677\n",
            "           1       0.25      0.60      0.35       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.59      0.69      0.60      1883\n",
            "weighted avg       0.86      0.76      0.80      1883\n",
            "\n",
            "Epoch: 19 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028644 | Test Loss: 0.029002\n",
            "Test loss decreased (0.029338 --> 0.029002). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.81      0.87      1677\n",
            "           1       0.25      0.52      0.34       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.59      0.67      0.60      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 20 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028376 | Test Loss: 0.029365\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.82      0.87      1677\n",
            "           1       0.26      0.52      0.35       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.67      0.61      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 21 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028205 | Test Loss: 0.029449\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.71      0.81      1677\n",
            "           1       0.21      0.64      0.32       206\n",
            "\n",
            "    accuracy                           0.70      1883\n",
            "   macro avg       0.58      0.67      0.56      1883\n",
            "weighted avg       0.86      0.70      0.75      1883\n",
            "\n",
            "Epoch: 22 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028154 | Test Loss: 0.029813\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.78      0.85      1677\n",
            "           1       0.24      0.55      0.33       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.59      0.67      0.59      1883\n",
            "weighted avg       0.86      0.76      0.80      1883\n",
            "\n",
            "Epoch: 23 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028173 | Test Loss: 0.029128\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.82      0.87      1677\n",
            "           1       0.26      0.51      0.35       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.67      0.61      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 24 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.028002 | Test Loss: 0.029373\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.69      0.80      1677\n",
            "           1       0.22      0.71      0.33       206\n",
            "\n",
            "    accuracy                           0.69      1883\n",
            "   macro avg       0.58      0.70      0.56      1883\n",
            "weighted avg       0.87      0.69      0.75      1883\n",
            "\n",
            "Epoch: 25 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.027858 | Test Loss: 0.028937\n",
            "Test loss decreased (0.029002 --> 0.028937). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.71      0.81      1677\n",
            "           1       0.22      0.66      0.33       206\n",
            "\n",
            "    accuracy                           0.71      1883\n",
            "   macro avg       0.58      0.69      0.57      1883\n",
            "weighted avg       0.87      0.71      0.76      1883\n",
            "\n",
            "Epoch: 26 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.027271 | Test Loss: 0.029259\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.94      0.93      1677\n",
            "           1       0.36      0.28      0.32       206\n",
            "\n",
            "    accuracy                           0.87      1883\n",
            "   macro avg       0.64      0.61      0.62      1883\n",
            "weighted avg       0.85      0.87      0.86      1883\n",
            "\n",
            "Epoch: 27 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.027264 | Test Loss: 0.033545\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.81      0.87      1677\n",
            "           1       0.26      0.54      0.36       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.60      0.68      0.61      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 28 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.027278 | Test Loss: 0.028668\n",
            "Test loss decreased (0.028937 --> 0.028668). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.64      0.77      1677\n",
            "           1       0.20      0.72      0.31       206\n",
            "\n",
            "    accuracy                           0.65      1883\n",
            "   macro avg       0.57      0.68      0.54      1883\n",
            "weighted avg       0.87      0.65      0.72      1883\n",
            "\n",
            "Epoch: 29 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.027023 | Test Loss: 0.030229\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.80      0.86      1677\n",
            "           1       0.25      0.56      0.35       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.60      0.68      0.61      1883\n",
            "weighted avg       0.86      0.77      0.81      1883\n",
            "\n",
            "Epoch: 30 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026782 | Test Loss: 0.029089\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.92      0.92      1677\n",
            "           1       0.31      0.31      0.31       206\n",
            "\n",
            "    accuracy                           0.85      1883\n",
            "   macro avg       0.61      0.61      0.61      1883\n",
            "weighted avg       0.85      0.85      0.85      1883\n",
            "\n",
            "Epoch: 31 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026725 | Test Loss: 0.032318\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.66      0.78      1677\n",
            "           1       0.21      0.73      0.33       206\n",
            "\n",
            "    accuracy                           0.67      1883\n",
            "   macro avg       0.58      0.70      0.56      1883\n",
            "weighted avg       0.87      0.67      0.73      1883\n",
            "\n",
            "Epoch: 32 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026344 | Test Loss: 0.029108\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.80      0.86      1677\n",
            "           1       0.25      0.54      0.34       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.59      0.67      0.60      1883\n",
            "weighted avg       0.86      0.77      0.81      1883\n",
            "\n",
            "Epoch: 33 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026256 | Test Loss: 0.029369\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.86      0.89      1677\n",
            "           1       0.29      0.47      0.36       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.61      0.66      0.63      1883\n",
            "weighted avg       0.86      0.82      0.84      1883\n",
            "\n",
            "Epoch: 34 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026447 | Test Loss: 0.030064\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.88      0.90      1677\n",
            "           1       0.29      0.39      0.34       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.61      0.64      0.62      1883\n",
            "weighted avg       0.85      0.83      0.84      1883\n",
            "\n",
            "Epoch: 35 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026364 | Test Loss: 0.030767\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.93      0.92      1677\n",
            "           1       0.32      0.27      0.29       206\n",
            "\n",
            "    accuracy                           0.86      1883\n",
            "   macro avg       0.62      0.60      0.61      1883\n",
            "weighted avg       0.85      0.86      0.85      1883\n",
            "\n",
            "Epoch: 36 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026050 | Test Loss: 0.036651\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.82      0.88      1677\n",
            "           1       0.27      0.54      0.36       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.68      0.62      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 37 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026074 | Test Loss: 0.029283\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.81      0.87      1677\n",
            "           1       0.26      0.55      0.35       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.60      0.68      0.61      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 38 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026134 | Test Loss: 0.028558\n",
            "Test loss decreased (0.028668 --> 0.028558). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.80      0.86      1677\n",
            "           1       0.25      0.55      0.35       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.59      0.67      0.60      1883\n",
            "weighted avg       0.86      0.77      0.81      1883\n",
            "\n",
            "Epoch: 39 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025712 | Test Loss: 0.030006\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.81      0.87      1677\n",
            "           1       0.28      0.59      0.38       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.61      0.70      0.63      1883\n",
            "weighted avg       0.87      0.79      0.82      1883\n",
            "\n",
            "Epoch: 40 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025729 | Test Loss: 0.028373\n",
            "Test loss decreased (0.028558 --> 0.028373). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.59      0.73      1677\n",
            "           1       0.20      0.82      0.32       206\n",
            "\n",
            "    accuracy                           0.61      1883\n",
            "   macro avg       0.58      0.70      0.52      1883\n",
            "weighted avg       0.88      0.61      0.68      1883\n",
            "\n",
            "Epoch: 41 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025756 | Test Loss: 0.030150\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.96      0.93      1677\n",
            "           1       0.42      0.24      0.31       206\n",
            "\n",
            "    accuracy                           0.88      1883\n",
            "   macro avg       0.67      0.60      0.62      1883\n",
            "weighted avg       0.86      0.88      0.87      1883\n",
            "\n",
            "Epoch: 42 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026105 | Test Loss: 0.032473\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.88      0.90      1677\n",
            "           1       0.30      0.43      0.35       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.61      0.65      0.63      1883\n",
            "weighted avg       0.86      0.83      0.84      1883\n",
            "\n",
            "Epoch: 43 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025591 | Test Loss: 0.030584\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.77      0.85      1677\n",
            "           1       0.25      0.63      0.36       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.60      0.70      0.60      1883\n",
            "weighted avg       0.87      0.75      0.79      1883\n",
            "\n",
            "Epoch: 44 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.026049 | Test Loss: 0.029092\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.73      0.82      1677\n",
            "           1       0.24      0.68      0.35       206\n",
            "\n",
            "    accuracy                           0.72      1883\n",
            "   macro avg       0.59      0.71      0.59      1883\n",
            "weighted avg       0.87      0.72      0.77      1883\n",
            "\n",
            "Epoch: 45 | Epoch Time: 0m 14s\n",
            "\tTraining Loss: 0.025174 | Test Loss: 0.028666\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.68      0.79      1677\n",
            "           1       0.22      0.74      0.34       206\n",
            "\n",
            "    accuracy                           0.69      1883\n",
            "   macro avg       0.59      0.71      0.57      1883\n",
            "weighted avg       0.87      0.69      0.74      1883\n",
            "\n",
            "Epoch: 46 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025780 | Test Loss: 0.028073\n",
            "Test loss decreased (0.028373 --> 0.028073). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.90      0.91      1677\n",
            "           1       0.34      0.41      0.37       206\n",
            "\n",
            "    accuracy                           0.85      1883\n",
            "   macro avg       0.63      0.65      0.64      1883\n",
            "weighted avg       0.86      0.85      0.85      1883\n",
            "\n",
            "Epoch: 47 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025511 | Test Loss: 0.032020\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.85      0.89      1677\n",
            "           1       0.29      0.50      0.37       206\n",
            "\n",
            "    accuracy                           0.81      1883\n",
            "   macro avg       0.61      0.68      0.63      1883\n",
            "weighted avg       0.86      0.81      0.83      1883\n",
            "\n",
            "Epoch: 48 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025659 | Test Loss: 0.028117\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.67      0.79      1677\n",
            "           1       0.22      0.76      0.34       206\n",
            "\n",
            "    accuracy                           0.68      1883\n",
            "   macro avg       0.59      0.71      0.56      1883\n",
            "weighted avg       0.88      0.68      0.74      1883\n",
            "\n",
            "Epoch: 49 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.024919 | Test Loss: 0.029036\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.81      0.87      1677\n",
            "           1       0.26      0.52      0.34       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.59      0.67      0.61      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 50 | Epoch Time: 0m 15s\n",
            "\tTraining Loss: 0.025602 | Test Loss: 0.028891\n",
            "======================================================================\n"
          ]
        }
      ],
      "source": [
        "model_lstm = LSTM_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "test_min_loss = np.inf\n",
        "criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weight)).to(device)\n",
        "optimizer_lstm = optim.Adam(model_lstm.parameters(), lr=1e-3)\n",
        "    \n",
        "for epoch in range(N_EPOCHS):\n",
        "    start_time = time.time()\n",
        "    model_lstm.train()\n",
        "    train_loss = 0.0\n",
        "    test_loss = 0.0\n",
        "    for inputs, target in train_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        model_lstm.zero_grad()\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model_lstm(inputs)\n",
        "        loss = criterion(output, target)\n",
        "        loss.backward()\n",
        "        train_loss += loss.item()\n",
        "        optimizer_lstm.step()\n",
        "\n",
        "    y_pred_list = []\n",
        "    y_targ_list = []\n",
        "    model_lstm.eval()\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model_lstm(inputs)\n",
        "        loss = criterion(output, target)\n",
        "        test_loss += loss.item()\n",
        "        _, y_test_pred = torch.max(output, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "    \n",
        "\n",
        "    if(epoch%1 == 0):\n",
        "        y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "        y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "        y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "        y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "        print(classification_report(y_targ_list, y_pred_list))\n",
        "\n",
        "    train_loss = train_loss / len(train_loader.dataset)\n",
        "    test_loss = test_loss / len(test_loader.dataset)\n",
        "\n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "\n",
        "    print(f\"Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s\")\n",
        "    print(\"\\tTraining Loss: {:.6f} | Test Loss: {:.6f}\".format(train_loss, test_loss))\n",
        "    if test_loss <= test_min_loss:\n",
        "        print(\"Test loss decreased ({:.6f} --> {:.6f}). Saving model...\\n\".format(test_min_loss, test_loss))\n",
        "        torch.save(model_lstm.state_dict(), \"models/lstm_bert_model.pt\")\n",
        "        test_min_loss = test_loss\n",
        "    print(\"=\"*70)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YjycJSZAQNhj",
        "outputId": "4993626f-bd94-4f77-eb2a-fe27d17f4410"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.68      0.79      1677\n",
            "           1       0.22      0.74      0.34       206\n",
            "\n",
            "    accuracy                           0.69      1883\n",
            "   macro avg       0.59      0.71      0.57      1883\n",
            "weighted avg       0.87      0.69      0.74      1883\n",
            "\n"
          ]
        }
      ],
      "source": [
        "y_pred_list = []\n",
        "y_targ_list = []\n",
        "model = LSTM_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "model.load_state_dict(torch.load(\"models/lstm_bert_model.pt\"))\n",
        "model.eval()\n",
        "\n",
        "with torch.no_grad():\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        y_test_pred = model(inputs)\n",
        "        _, y_test_pred = torch.max(y_test_pred, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "\n",
        "y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "print(classification_report(y_targ_list, y_pred_list))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U9-Lxl43XWZB"
      },
      "source": [
        "#**Bi-LSTM**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "q6VmQjWiTArv"
      },
      "outputs": [],
      "source": [
        "class Bi_LSTM_Network(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
        "\n",
        "        super(Bi_LSTM_Network, self).__init__()\n",
        "\n",
        "        # Number of hidden dimensions\n",
        "        self.hidden_dim = hidden_dim\n",
        "\n",
        "        # RNN\n",
        "        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=1, batch_first=True, bidirectional=True)\n",
        "\n",
        "        # Readout layer\n",
        "        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        # Initialize hidden state with zeros\n",
        "        h0 = Variable(torch.zeros(1 * 2, x.size(0), self.hidden_dim)).to(device)\n",
        "        c0 = Variable(torch.zeros(1 * 2, x.size(0), self.hidden_dim)).to(device)\n",
        "\n",
        "        # One time step\n",
        "        out, (hn, cn) = self.lstm(x, (h0, c0))\n",
        "        out = self.fc(out[:, -1, :])\n",
        "\n",
        "        return out\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a_JNSTKhe7Me",
        "outputId": "188ad70e-887f-4e80-d368-57a61f32fbcc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.90      0.95      0.93      1677\n",
            "           1       0.32      0.18      0.23       206\n",
            "\n",
            "    accuracy                           0.87      1883\n",
            "   macro avg       0.61      0.57      0.58      1883\n",
            "weighted avg       0.84      0.87      0.85      1883\n",
            "\n",
            "Epoch: 01 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.033636 \\Test Loss: 0.036043\n",
            "Test loss decreased (inf --> 0.036043). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.87      0.90      1677\n",
            "           1       0.26      0.35      0.30       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.59      0.61      0.60      1883\n",
            "weighted avg       0.84      0.82      0.83      1883\n",
            "\n",
            "Epoch: 02 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.032285 \\Test Loss: 0.031966\n",
            "Test loss decreased (0.036043 --> 0.031966). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.84      0.88      1677\n",
            "           1       0.24      0.41      0.30       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.58      0.63      0.59      1883\n",
            "weighted avg       0.85      0.80      0.82      1883\n",
            "\n",
            "Epoch: 03 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.031601 \\Test Loss: 0.031451\n",
            "Test loss decreased (0.031966 --> 0.031451). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.83      0.88      1677\n",
            "           1       0.25      0.47      0.33       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.59      0.65      0.60      1883\n",
            "weighted avg       0.85      0.79      0.82      1883\n",
            "\n",
            "Epoch: 04 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.031163 \\Test Loss: 0.030824\n",
            "Test loss decreased (0.031451 --> 0.030824). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.77      0.85      1677\n",
            "           1       0.22      0.53      0.32       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.58      0.65      0.58      1883\n",
            "weighted avg       0.85      0.75      0.79      1883\n",
            "\n",
            "Epoch: 05 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.031021 \\Test Loss: 0.030652\n",
            "Test loss decreased (0.030824 --> 0.030652). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.80      0.86      1677\n",
            "           1       0.24      0.53      0.33       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.59      0.66      0.60      1883\n",
            "weighted avg       0.86      0.77      0.80      1883\n",
            "\n",
            "Epoch: 06 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.030540 \\Test Loss: 0.030228\n",
            "Test loss decreased (0.030652 --> 0.030228). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.79      0.85      1677\n",
            "           1       0.24      0.55      0.33       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.59      0.67      0.59      1883\n",
            "weighted avg       0.86      0.76      0.80      1883\n",
            "\n",
            "Epoch: 07 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.030131 \\Test Loss: 0.030033\n",
            "Test loss decreased (0.030228 --> 0.030033). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.80      0.86      1677\n",
            "           1       0.24      0.50      0.32       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.58      0.65      0.59      1883\n",
            "weighted avg       0.85      0.77      0.80      1883\n",
            "\n",
            "Epoch: 08 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.030124 \\Test Loss: 0.029984\n",
            "Test loss decreased (0.030033 --> 0.029984). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.83      0.88      1677\n",
            "           1       0.26      0.49      0.34       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.59      0.66      0.61      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 09 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.029959 \\Test Loss: 0.030040\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.86      0.89      1677\n",
            "           1       0.28      0.45      0.34       206\n",
            "\n",
            "    accuracy                           0.81      1883\n",
            "   macro avg       0.60      0.65      0.62      1883\n",
            "weighted avg       0.86      0.81      0.83      1883\n",
            "\n",
            "Epoch: 10 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.029742 \\Test Loss: 0.029943\n",
            "Test loss decreased (0.029984 --> 0.029943). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.84      0.88      1677\n",
            "           1       0.26      0.45      0.33       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.59      0.65      0.60      1883\n",
            "weighted avg       0.85      0.80      0.82      1883\n",
            "\n",
            "Epoch: 11 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.029496 \\Test Loss: 0.029887\n",
            "Test loss decreased (0.029943 --> 0.029887). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.94      0.92      1677\n",
            "           1       0.31      0.23      0.27       206\n",
            "\n",
            "    accuracy                           0.86      1883\n",
            "   macro avg       0.61      0.58      0.59      1883\n",
            "weighted avg       0.84      0.86      0.85      1883\n",
            "\n",
            "Epoch: 12 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.029227 \\Test Loss: 0.032293\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.77      0.84      1677\n",
            "           1       0.23      0.57      0.33       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.58      0.67      0.58      1883\n",
            "weighted avg       0.86      0.74      0.79      1883\n",
            "\n",
            "Epoch: 13 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.029282 \\Test Loss: 0.029842\n",
            "Test loss decreased (0.029887 --> 0.029842). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.89      0.91      1677\n",
            "           1       0.31      0.41      0.35       206\n",
            "\n",
            "    accuracy                           0.84      1883\n",
            "   macro avg       0.62      0.65      0.63      1883\n",
            "weighted avg       0.86      0.84      0.85      1883\n",
            "\n",
            "Epoch: 14 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.029187 \\Test Loss: 0.030186\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.81      0.87      1677\n",
            "           1       0.26      0.54      0.35       206\n",
            "\n",
            "    accuracy                           0.78      1883\n",
            "   macro avg       0.60      0.67      0.61      1883\n",
            "weighted avg       0.86      0.78      0.81      1883\n",
            "\n",
            "Epoch: 15 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.028783 \\Test Loss: 0.029741\n",
            "Test loss decreased (0.029842 --> 0.029741). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.68      0.79      1677\n",
            "           1       0.21      0.69      0.32       206\n",
            "\n",
            "    accuracy                           0.68      1883\n",
            "   macro avg       0.58      0.69      0.56      1883\n",
            "weighted avg       0.87      0.68      0.74      1883\n",
            "\n",
            "Epoch: 16 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.028587 \\Test Loss: 0.029589\n",
            "Test loss decreased (0.029741 --> 0.029589). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.61      0.74      1677\n",
            "           1       0.19      0.74      0.30       206\n",
            "\n",
            "    accuracy                           0.62      1883\n",
            "   macro avg       0.57      0.68      0.52      1883\n",
            "weighted avg       0.87      0.62      0.69      1883\n",
            "\n",
            "Epoch: 17 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.028367 \\Test Loss: 0.030212\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.84      0.88      1677\n",
            "           1       0.27      0.49      0.35       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.60      0.66      0.61      1883\n",
            "weighted avg       0.86      0.80      0.82      1883\n",
            "\n",
            "Epoch: 18 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.028637 \\Test Loss: 0.029815\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.87      0.90      1677\n",
            "           1       0.29      0.42      0.34       206\n",
            "\n",
            "    accuracy                           0.82      1883\n",
            "   macro avg       0.61      0.64      0.62      1883\n",
            "weighted avg       0.85      0.82      0.84      1883\n",
            "\n",
            "Epoch: 19 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.028093 \\Test Loss: 0.029634\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.75      0.84      1677\n",
            "           1       0.23      0.60      0.33       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.58      0.68      0.59      1883\n",
            "weighted avg       0.86      0.74      0.78      1883\n",
            "\n",
            "Epoch: 20 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.028316 \\Test Loss: 0.028894\n",
            "Test loss decreased (0.029589 --> 0.028894). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.57      0.71      1677\n",
            "           1       0.18      0.76      0.29       206\n",
            "\n",
            "    accuracy                           0.59      1883\n",
            "   macro avg       0.56      0.66      0.50      1883\n",
            "weighted avg       0.87      0.59      0.66      1883\n",
            "\n",
            "Epoch: 22 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.027998 \\Test Loss: 0.030761\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.75      0.84      1677\n",
            "           1       0.23      0.61      0.34       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.59      0.68      0.59      1883\n",
            "weighted avg       0.86      0.74      0.78      1883\n",
            "\n",
            "Epoch: 23 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.028074 \\Test Loss: 0.029135\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.82      0.87      1677\n",
            "           1       0.26      0.52      0.35       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.67      0.61      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 24 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.027433 \\Test Loss: 0.029336\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.74      0.83      1677\n",
            "           1       0.22      0.61      0.33       206\n",
            "\n",
            "    accuracy                           0.73      1883\n",
            "   macro avg       0.58      0.67      0.58      1883\n",
            "weighted avg       0.86      0.73      0.77      1883\n",
            "\n",
            "Epoch: 25 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026956 \\Test Loss: 0.029640\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.75      0.84      1677\n",
            "           1       0.23      0.60      0.33       206\n",
            "\n",
            "    accuracy                           0.74      1883\n",
            "   macro avg       0.59      0.68      0.59      1883\n",
            "weighted avg       0.86      0.74      0.78      1883\n",
            "\n",
            "Epoch: 26 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.027486 \\Test Loss: 0.029031\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.74      0.83      1677\n",
            "           1       0.23      0.63      0.34       206\n",
            "\n",
            "    accuracy                           0.73      1883\n",
            "   macro avg       0.59      0.68      0.58      1883\n",
            "weighted avg       0.86      0.73      0.78      1883\n",
            "\n",
            "Epoch: 27 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.027437 \\Test Loss: 0.028520\n",
            "Test loss decreased (0.028894 --> 0.028520). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.88      0.90      1677\n",
            "           1       0.30      0.42      0.35       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.61      0.65      0.63      1883\n",
            "weighted avg       0.86      0.83      0.84      1883\n",
            "\n",
            "Epoch: 28 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.027251 \\Test Loss: 0.029214\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.84      0.88      1677\n",
            "           1       0.27      0.48      0.35       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.60      0.66      0.62      1883\n",
            "weighted avg       0.86      0.80      0.83      1883\n",
            "\n",
            "Epoch: 29 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.027361 \\Test Loss: 0.029224\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.89      0.91      1677\n",
            "           1       0.31      0.41      0.35       206\n",
            "\n",
            "    accuracy                           0.83      1883\n",
            "   macro avg       0.62      0.65      0.63      1883\n",
            "weighted avg       0.86      0.83      0.84      1883\n",
            "\n",
            "Epoch: 30 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026730 \\Test Loss: 0.030162\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.84      0.88      1677\n",
            "           1       0.28      0.50      0.36       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.60      0.67      0.62      1883\n",
            "weighted avg       0.86      0.80      0.83      1883\n",
            "\n",
            "Epoch: 31 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026728 \\Test Loss: 0.029454\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.72      0.82      1677\n",
            "           1       0.23      0.67      0.34       206\n",
            "\n",
            "    accuracy                           0.72      1883\n",
            "   macro avg       0.59      0.70      0.58      1883\n",
            "weighted avg       0.87      0.72      0.77      1883\n",
            "\n",
            "Epoch: 32 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026231 \\Test Loss: 0.028361\n",
            "Test loss decreased (0.028520 --> 0.028361). Saving model...\n",
            "\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.64      0.77      1677\n",
            "           1       0.20      0.73      0.31       206\n",
            "\n",
            "    accuracy                           0.65      1883\n",
            "   macro avg       0.58      0.69      0.54      1883\n",
            "weighted avg       0.87      0.65      0.72      1883\n",
            "\n",
            "Epoch: 33 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026419 \\Test Loss: 0.030041\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.79      0.86      1677\n",
            "           1       0.25      0.58      0.35       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.60      0.68      0.61      1883\n",
            "weighted avg       0.86      0.77      0.80      1883\n",
            "\n",
            "Epoch: 34 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026238 \\Test Loss: 0.028506\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.84      0.88      1677\n",
            "           1       0.27      0.49      0.35       206\n",
            "\n",
            "    accuracy                           0.80      1883\n",
            "   macro avg       0.60      0.66      0.62      1883\n",
            "weighted avg       0.86      0.80      0.83      1883\n",
            "\n",
            "Epoch: 35 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026355 \\Test Loss: 0.028835\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.77      0.85      1677\n",
            "           1       0.24      0.59      0.34       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.59      0.68      0.60      1883\n",
            "weighted avg       0.86      0.75      0.79      1883\n",
            "\n",
            "Epoch: 36 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026291 \\Test Loss: 0.028954\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.71      0.81      1677\n",
            "           1       0.22      0.66      0.33       206\n",
            "\n",
            "    accuracy                           0.70      1883\n",
            "   macro avg       0.58      0.68      0.57      1883\n",
            "weighted avg       0.86      0.70      0.76      1883\n",
            "\n",
            "Epoch: 37 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.025650 \\Test Loss: 0.028925\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.75      0.83      1677\n",
            "           1       0.23      0.63      0.34       206\n",
            "\n",
            "    accuracy                           0.73      1883\n",
            "   macro avg       0.59      0.69      0.59      1883\n",
            "weighted avg       0.87      0.73      0.78      1883\n",
            "\n",
            "Epoch: 38 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.025636 \\Test Loss: 0.029267\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.78      0.85      1677\n",
            "           1       0.24      0.57      0.34       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.59      0.68      0.60      1883\n",
            "weighted avg       0.86      0.76      0.80      1883\n",
            "\n",
            "Epoch: 39 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.026076 \\Test Loss: 0.029510\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.64      0.77      1677\n",
            "           1       0.20      0.74      0.32       206\n",
            "\n",
            "    accuracy                           0.65      1883\n",
            "   macro avg       0.58      0.69      0.54      1883\n",
            "weighted avg       0.87      0.65      0.72      1883\n",
            "\n",
            "Epoch: 40 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.025948 \\Test Loss: 0.029014\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.78      0.85      1677\n",
            "           1       0.24      0.58      0.34       206\n",
            "\n",
            "    accuracy                           0.76      1883\n",
            "   macro avg       0.59      0.68      0.60      1883\n",
            "weighted avg       0.86      0.76      0.79      1883\n",
            "\n",
            "Epoch: 41 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.025789 \\Test Loss: 0.029175\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.77      0.84      1677\n",
            "           1       0.24      0.60      0.34       206\n",
            "\n",
            "    accuracy                           0.75      1883\n",
            "   macro avg       0.59      0.68      0.59      1883\n",
            "weighted avg       0.86      0.75      0.79      1883\n",
            "\n",
            "Epoch: 42 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.025456 \\Test Loss: 0.028831\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.89      0.91      1677\n",
            "           1       0.30      0.37      0.33       206\n",
            "\n",
            "    accuracy                           0.84      1883\n",
            "   macro avg       0.61      0.63      0.62      1883\n",
            "weighted avg       0.85      0.84      0.84      1883\n",
            "\n",
            "Epoch: 43 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.024917 \\Test Loss: 0.031604\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.64      0.76      1677\n",
            "           1       0.20      0.75      0.32       206\n",
            "\n",
            "    accuracy                           0.65      1883\n",
            "   macro avg       0.58      0.69      0.54      1883\n",
            "weighted avg       0.87      0.65      0.71      1883\n",
            "\n",
            "Epoch: 44 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.025574 \\Test Loss: 0.029431\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.82      0.88      1677\n",
            "           1       0.27      0.54      0.36       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.60      0.68      0.62      1883\n",
            "weighted avg       0.86      0.79      0.82      1883\n",
            "\n",
            "Epoch: 45 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.025318 \\Test Loss: 0.029393\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.79      0.86      1677\n",
            "           1       0.25      0.58      0.35       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.60      0.68      0.61      1883\n",
            "weighted avg       0.86      0.77      0.80      1883\n",
            "\n",
            "Epoch: 46 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.024861 \\Test Loss: 0.029031\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.69      0.80      1677\n",
            "           1       0.22      0.70      0.33       206\n",
            "\n",
            "    accuracy                           0.69      1883\n",
            "   macro avg       0.58      0.69      0.56      1883\n",
            "weighted avg       0.87      0.69      0.75      1883\n",
            "\n",
            "Epoch: 47 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.024644 \\Test Loss: 0.029421\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.90      0.91      1677\n",
            "           1       0.33      0.38      0.35       206\n",
            "\n",
            "    accuracy                           0.85      1883\n",
            "   macro avg       0.62      0.64      0.63      1883\n",
            "weighted avg       0.86      0.85      0.85      1883\n",
            "\n",
            "Epoch: 48 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.024269 \\Test Loss: 0.030987\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.81      0.87      1677\n",
            "           1       0.27      0.57      0.37       206\n",
            "\n",
            "    accuracy                           0.79      1883\n",
            "   macro avg       0.61      0.69      0.62      1883\n",
            "weighted avg       0.87      0.79      0.82      1883\n",
            "\n",
            "Epoch: 49 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.024579 \\Test Loss: 0.029554\n",
            "======================================================================\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.79      0.86      1677\n",
            "           1       0.26      0.58      0.36       206\n",
            "\n",
            "    accuracy                           0.77      1883\n",
            "   macro avg       0.60      0.69      0.61      1883\n",
            "weighted avg       0.86      0.77      0.81      1883\n",
            "\n",
            "Epoch: 50 | Epoch Time: 0m 22s\n",
            "\tTraining Loss: 0.024371 \\Test Loss: 0.029155\n",
            "======================================================================\n"
          ]
        }
      ],
      "source": [
        "model_bilstm = Bi_LSTM_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "test_min_loss = np.inf\n",
        "criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weight)).to(device)\n",
        "optimizer_bilstm = optim.Adam(model_bilstm.parameters(), lr=1e-3) \n",
        "    \n",
        "for epoch in range(N_EPOCHS):\n",
        "    start_time = time.time()\n",
        "    model_bilstm.train()\n",
        "    train_loss = 0.0\n",
        "    test_loss = 0.0\n",
        "    for inputs, target in train_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        optimizer_bilstm.zero_grad()\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model_bilstm(inputs)\n",
        "        loss = criterion(output, target)\n",
        "        loss.backward()\n",
        "        train_loss += loss.item()\n",
        "        # torch.nn.utils.clip_grad_norm_(model_bilstm.parameters(), max_norm=4.0, norm_type=2) # try\n",
        "        optimizer_bilstm.step()\n",
        "\n",
        "    y_pred_list = []\n",
        "    y_targ_list = []\n",
        "    model_bilstm.eval()\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        output = model_bilstm(inputs)\n",
        "        loss = criterion(output, target)\n",
        "        test_loss += loss.item()\n",
        "        _, y_test_pred = torch.max(output, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "    \n",
        "\n",
        "    if(epoch%1 == 0):\n",
        "        y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "        y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "        y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "        y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "        print(classification_report(y_targ_list, y_pred_list))\n",
        "\n",
        "    train_loss = train_loss / len(train_loader.dataset)\n",
        "    test_loss = test_loss / len(test_loader.dataset)\n",
        "\n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "\n",
        "    print(f\"Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s\")\n",
        "    print(\"\\tTraining Loss: {:.6f} \\Test Loss: {:.6f}\".format(train_loss, test_loss))\n",
        "    if test_loss <= test_min_loss:\n",
        "        print(\"Test loss decreased ({:.6f} --> {:.6f}). Saving model...\\n\".format(test_min_loss, test_loss))\n",
        "        torch.save(model_bilstm.state_dict(), \"./models/bilstm_bert_model.pt\")\n",
        "        test_min_loss = test_loss\n",
        "    print(\"=\"*70)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zV3vER2n0rWF",
        "outputId": "3737048f-90b8-4eb9-8d2e-cdad5c75a324"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.72      0.82      1677\n",
            "           1       0.23      0.67      0.34       206\n",
            "\n",
            "    accuracy                           0.72      1883\n",
            "   macro avg       0.59      0.70      0.58      1883\n",
            "weighted avg       0.87      0.72      0.77      1883\n",
            "\n"
          ]
        }
      ],
      "source": [
        "y_pred_list = []\n",
        "y_targ_list = []\n",
        "model = Bi_LSTM_Network(EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM).to(device)\n",
        "model.load_state_dict(torch.load(\"models/bilstm_bert_model.pt\"))\n",
        "model.eval()\n",
        "\n",
        "with torch.no_grad():\n",
        "    for inputs, target in test_loader:\n",
        "        inputs, target = inputs.to(device), target.to(device)\n",
        "        inputs = torch.squeeze(inputs, dim=1)\n",
        "        y_test_pred = model(inputs)\n",
        "        _, y_test_pred = torch.max(y_test_pred, 1)\n",
        "        y_pred_tag = y_test_pred\n",
        "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
        "        y_targ_list.append(target.cpu().numpy())\n",
        "\n",
        "y_pred_list = [x.squeeze().tolist() for x in y_pred_list]\n",
        "y_targ_list = [x.squeeze().tolist() for x in y_targ_list]\n",
        "y_pred_list = [x for sublist in y_pred_list for x in sublist]\n",
        "y_targ_list = [x for sublist in y_targ_list for x in sublist]\n",
        "\n",
        "print(classification_report(y_targ_list, y_pred_list))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Best Performance : GRU"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}