ranking_2methods.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ranking_2methods.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "zdoJX3Qi29Ff",
        "Im_q-sLgPcUc",
        "1EmGkDtzJA3v",
        "c1S-QCewZztK",
        "j3Zw1UoqbNmD",
        "ne7uZIceNgPD",
        "99qMTrLtarC-"
      ],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b457fa85adb84bed9b4e986068c4803e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1c13a706855043728731dc2d4a4d56b1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b643ad092fe64382a1561c8e9c08c3ff",
              "IPY_MODEL_a2fdcc8cb41e4d7088f8b650ccb378b2"
            ]
          }
        },
        "1c13a706855043728731dc2d4a4d56b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b643ad092fe64382a1561c8e9c08c3ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_01df45d164594cb5ae483d0e64d43321",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a30d9e5bcb1445749ce999d917ba62ad"
          }
        },
        "a2fdcc8cb41e4d7088f8b650ccb378b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe0e0bbf19bb47e298d6acad3b22d846",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:02&lt;00:00, 174kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c25c2ff703a340f38c2196c217a4ed0c"
          }
        },
        "01df45d164594cb5ae483d0e64d43321": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a30d9e5bcb1445749ce999d917ba62ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe0e0bbf19bb47e298d6acad3b22d846": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c25c2ff703a340f38c2196c217a4ed0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "75d299d7636e4ef9ad6177f770a65af0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7ace5dc1fb204177826b649ea18ea33c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b8bdcccacbcb4d9abe0a7f79bcfec7de",
              "IPY_MODEL_48f1603b8bda4e61b72e25f2c1996721"
            ]
          }
        },
        "7ace5dc1fb204177826b649ea18ea33c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8bdcccacbcb4d9abe0a7f79bcfec7de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_81e5556e39754bc4afc6d464473b3159",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0116145c24524f5682e543046e43b204"
          }
        },
        "48f1603b8bda4e61b72e25f2c1996721": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_db2cd49c426c430c9eddffc4b188896a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:01&lt;00:00, 18.2B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e8e21bc428294995a598550fe085be49"
          }
        },
        "81e5556e39754bc4afc6d464473b3159": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0116145c24524f5682e543046e43b204": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db2cd49c426c430c9eddffc4b188896a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e8e21bc428294995a598550fe085be49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "73tCzNla103Q",
        "outputId": "06be0dea-9c90-4bed-c2b0-3d93b13c6f21"
      },
      "source": [
        "!pip install transformers\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 7.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 38.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 39.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.45 tokenizers-0.10.2 transformers-4.5.1\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qNTlgcdrUJAB"
      },
      "source": [
        "import sys\n",
        "sys.path.insert(0, \"//\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wD-soWFU3FcN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6b165dd0-dc8d-440e-972b-dea13bbcc24b"
      },
      "source": [
        "import numpy as np\n",
        "import time\n",
        "import torch\n",
        "import os\n",
        "import torch.nn as nn\n",
        "import time\n",
        "import gc\n",
        "import copy\n",
        "import re\n",
        "import pandas as pd\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from transformers import DistilBertTokenizer, DistilBertForSequenceClassification\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from sklearn.metrics import accuracy_score\n",
        "# device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "print(sys.path)\n",
        "from content.data_extractor import DataExtractor\n",
        "from content.data_extractor import TorchDataset"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['//', '', '/content', '/env/python', '/usr/lib/python37.zip', '/usr/lib/python3.7', '/usr/lib/python3.7/lib-dynload', '/usr/local/lib/python3.7/dist-packages', '/usr/lib/python3/dist-packages', '/usr/local/lib/python3.7/dist-packages/IPython/extensions', '/root/.ipython']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_VKXkv0ML1iy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a211cdbe-1d75-49f1-9b39-6c530b7e2f4d"
      },
      "source": [
        "torch.cuda.is_available()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zdoJX3Qi29Ff"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jj8lLiUlC95Z",
        "outputId": "509ba83c-4ddf-4b96-e99d-aefc7a2aded0"
      },
      "source": [
        "dataset = TorchDataset(\n",
        "    file_name=\"./data/diverse.triplets.train.tsv\",\n",
        "    queries_path=\"./data/diverse.queries.all.tsv\",\n",
        "    passages_path=\"./data/diverse.passages.all.tsv\",\n",
        ")\n",
        "type(dataset)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "content.data_extractor.TorchDataset"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fCyRXxPOUdHV",
        "outputId": "0d83b69e-82e5-44f6-abd6-f2375be7df6c"
      },
      "source": [
        "dataset.__getitem__(9000)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('heir apparent main character',\n",
              " \"Penrod is Princess Janine's guard. He is protective, loyal, strong, a good listener, and respectful. If I had to choose one of the three main characters in Heir Apparent to be hang out with, I would choose Giannine. She is brave and seems to always have her friends' and family's back.\",\n",
              " 'A short story (usually) has one main character. It is not unusual for there to be only one character. There can be other SECONDARY characters, but all in a short story has a very limited number of characters. You are correct: central character and main character are the same thing.')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bUa39hyZ3Jvi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 132,
          "referenced_widgets": [
            "b457fa85adb84bed9b4e986068c4803e",
            "1c13a706855043728731dc2d4a4d56b1",
            "b643ad092fe64382a1561c8e9c08c3ff",
            "a2fdcc8cb41e4d7088f8b650ccb378b2",
            "01df45d164594cb5ae483d0e64d43321",
            "a30d9e5bcb1445749ce999d917ba62ad",
            "fe0e0bbf19bb47e298d6acad3b22d846",
            "c25c2ff703a340f38c2196c217a4ed0c",
            "75d299d7636e4ef9ad6177f770a65af0",
            "7ace5dc1fb204177826b649ea18ea33c",
            "b8bdcccacbcb4d9abe0a7f79bcfec7de",
            "48f1603b8bda4e61b72e25f2c1996721",
            "81e5556e39754bc4afc6d464473b3159",
            "0116145c24524f5682e543046e43b204",
            "db2cd49c426c430c9eddffc4b188896a",
            "e8e21bc428294995a598550fe085be49"
          ]
        },
        "outputId": "a3a6e775-8bfa-46ea-a254-e15bb3c23262"
      },
      "source": [
        "from transformers import (\n",
        "    DistilBertTokenizerFast,\n",
        "    DistilBertForSequenceClassification,\n",
        "    DistilBertForMaskedLM,\n",
        ")\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained(\"distilbert-base-uncased\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b457fa85adb84bed9b4e986068c4803e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "75d299d7636e4ef9ad6177f770a65af0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rvzb7KNn3JrD"
      },
      "source": [
        "batch_size = 8\n",
        "train_dataset = TorchDataset(\n",
        "    file_name=\"./data/diverse.triplets.train.tsv\",\n",
        "    queries_path=\"./data/diverse.queries.all.tsv\",\n",
        "    passages_path=\"./data/diverse.passages.all.tsv\",\n",
        ")\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "dev_dataset = TorchDataset(\n",
        "    file_name=\"./data/diverse.triplets.dev.tsv\",\n",
        "    queries_path=\"./data/diverse.queries.all.tsv\",\n",
        "    passages_path=\"./data/diverse.passages.all.tsv\",\n",
        ")\n",
        "dev_dataloader = DataLoader(dev_dataset, batch_size=batch_size, shuffle=False)\n",
        "\n",
        "test_dataset = TorchDataset(\n",
        "    file_name=\"./data/diverse.triplets.test.tsv\",\n",
        "    queries_path=\"./data/diverse.queries.all.tsv\",\n",
        "    passages_path=\"./data/diverse.passages.all.tsv\",\n",
        ")\n",
        "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5-NkiBtBZvs0"
      },
      "source": [
        "# Two approaches:\n",
        "- A binary classier \n",
        "- A model generate vector representation for queries and documents\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hK5yMyPWXdKi"
      },
      "source": [
        "model_path = '/content/drive/MyDrive/reranking/'\n",
        "NUM_EPOCHS = 1\n",
        "LEARNING_RATE=0.00003"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Im_q-sLgPcUc"
      },
      "source": [
        "## Classification-based approach\n",
        "- Data: query, positive_document, negative_document\n",
        "- Training with entropy loss as a classification problem; replicate the CMU paper\n",
        "- Inference: if a document is relevant or not relevant"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7xzcDCIAbTTx"
      },
      "source": [
        "### Train/Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rWelsbj-Sx5F",
        "outputId": "ca33087d-3e5c-4e89-f42e-233eddc3d555"
      },
      "source": [
        "NUM_EPOCHS = 1\n",
        "LEARNING_RATE = 0.00003\n",
        "\n",
        "model = DistilBertForSequenceClassification.from_pretrained(\"distilbert-base-uncased\",num_labels=2)\n",
        "model.to(device)\n",
        "model.train()\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "def predict(inputs, model, tokenizer, labels):\n",
        "    encodings = tokenizer(\n",
        "        inputs,\n",
        "        return_tensors=\"pt\",\n",
        "        truncation=True,\n",
        "        padding=True,\n",
        "        max_length=512,\n",
        "    )\n",
        "    ids, masks = encodings[\"input_ids\"], encodings[\"attention_mask\"]\n",
        "    outputs = model(ids.to(device), masks.to(device), labels=labels.to(device))\n",
        "    return outputs\n",
        "\n",
        "\n",
        "dataloader = train_dataloader\n",
        "N = len(dataloader)\n",
        "lowest_loss = float(\"inf\")\n",
        "start = time.time()\n",
        "learning_curve_y = []\n",
        "learning_curve_x = []\n",
        "\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    epoch_loss = 0\n",
        "    for i, (queries, pos_docs, neg_docs) in enumerate(dataloader):\n",
        "\n",
        "        optimizer.zero_grad()  # set gradient to zero\n",
        "\n",
        "        queries = list(queries) * 2  # 2*B\n",
        "        docs = list(pos_docs) + list(neg_docs)\n",
        "\n",
        "        labels = torch.cat([torch.ones(len(pos_docs)), \n",
        "                            torch.zeros(len(neg_docs))]).long().to(device)  # 2*batch,\n",
        "\n",
        "        outputs = predict(\n",
        "            inputs=list(zip(queries, docs)),\n",
        "            model=model,\n",
        "            tokenizer=tokenizer,\n",
        "            labels=labels,\n",
        "        )\n",
        "\n",
        "        loss = outputs.loss\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        epoch_loss += float(loss)\n",
        "\n",
        "        if i % 10 == 0:\n",
        "            elapsed_time = time.time() - start\n",
        "            remaining_time = elapsed_time * (1 / (i + 1) * N - 1)\n",
        "            print(\n",
        "                f\"{i}: remaining time: {remaining_time:.1f} | est. epoch loss: {epoch_loss / (i + 1):.4f}\"\n",
        "            )\n",
        "\n",
        "        if i % 10 == 0:\n",
        "            with torch.no_grad():\n",
        "                correct = total = 0\n",
        "                val_start = time.time()\n",
        "                for dq, dp, dn in dev_dataloader:\n",
        "                    queries = list(dq) * 2  # 2*B\n",
        "                    docs = list(dp) + list(dn)\n",
        "                    labels = torch.cat([torch.ones(len(dp)), \n",
        "                             torch.zeros(len(dn))]).long().to(device)\n",
        "                    outputs = predict(\n",
        "                        inputs=list(zip(queries, docs)),\n",
        "                        model=model,\n",
        "                        tokenizer=tokenizer,\n",
        "                        labels=labels\n",
        "                    )\n",
        "\n",
        "                    predicted_classes = outputs.logits.argmax(dim=-1)\n",
        "                    correct += (labels == predicted_classes).sum()\n",
        "\n",
        "                    total += len(labels)\n",
        "                    if time.time() - val_start > 15:\n",
        "                        break\n",
        "                print(f\"{i}: est. validation accuracy: {correct/total:.4f}\")\n",
        "                learning_curve_y.append(correct/total)\n",
        "                learning_curve_x.append(i * batch_size)  # epoch normally \n",
        "\n",
        "        if (epoch_loss / (i + 1)) < lowest_loss:\n",
        "            torch.save(model.state_dict(), \"model1_best.pt\")\n",
        "            lowest_loss = epoch_loss / (i + 1)\n",
        "\n",
        "    print(f\"loss for epoch {epoch} is {epoch_loss}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "0: remaining time: 2041.4 | est. epoch loss: 0.6830\n",
            "0: est. validation accuracy: 0.5130\n",
            "10: remaining time: 13611.2 | est. epoch loss: 0.6953\n",
            "10: est. validation accuracy: 0.5748\n",
            "20: remaining time: 14033.5 | est. epoch loss: 0.6946\n",
            "20: est. validation accuracy: 0.5957\n",
            "30: remaining time: 14163.8 | est. epoch loss: 0.6888\n",
            "30: est. validation accuracy: 0.6323\n",
            "40: remaining time: 14237.9 | est. epoch loss: 0.6897\n",
            "40: est. validation accuracy: 0.6575\n",
            "50: remaining time: 14270.5 | est. epoch loss: 0.6873\n",
            "50: est. validation accuracy: 0.6750\n",
            "60: remaining time: 14256.6 | est. epoch loss: 0.6821\n",
            "60: est. validation accuracy: 0.6827\n",
            "70: remaining time: 15634.0 | est. epoch loss: 0.6750\n",
            "70: est. validation accuracy: 0.7084\n",
            "80: remaining time: 16299.3 | est. epoch loss: 0.6618\n",
            "80: est. validation accuracy: 0.7273\n",
            "90: remaining time: 17515.3 | est. epoch loss: 0.6436\n",
            "90: est. validation accuracy: 0.7253\n",
            "100: remaining time: 17921.5 | est. epoch loss: 0.6298\n",
            "100: est. validation accuracy: 0.7521\n",
            "110: remaining time: 18257.1 | est. epoch loss: 0.6156\n",
            "110: est. validation accuracy: 0.7586\n",
            "120: remaining time: 18879.7 | est. epoch loss: 0.6036\n",
            "120: est. validation accuracy: 0.7523\n",
            "130: remaining time: 19754.8 | est. epoch loss: 0.5872\n",
            "130: est. validation accuracy: 0.7529\n",
            "140: remaining time: 20208.0 | est. epoch loss: 0.5687\n",
            "140: est. validation accuracy: 0.7185\n",
            "150: remaining time: 20761.1 | est. epoch loss: 0.5586\n",
            "150: est. validation accuracy: 0.6913\n",
            "160: remaining time: 20739.7 | est. epoch loss: 0.5526\n",
            "160: est. validation accuracy: 0.7459\n",
            "170: remaining time: 21243.6 | est. epoch loss: 0.5412\n",
            "170: est. validation accuracy: 0.7548\n",
            "180: remaining time: 21575.4 | est. epoch loss: 0.5341\n",
            "180: est. validation accuracy: 0.7371\n",
            "190: remaining time: 21783.2 | est. epoch loss: 0.5273\n",
            "190: est. validation accuracy: 0.7480\n",
            "200: remaining time: 21967.5 | est. epoch loss: 0.5188\n",
            "200: est. validation accuracy: 0.7556\n",
            "210: remaining time: 22247.9 | est. epoch loss: 0.5128\n",
            "210: est. validation accuracy: 0.7690\n",
            "220: remaining time: 22562.1 | est. epoch loss: 0.5040\n",
            "220: est. validation accuracy: 0.7750\n",
            "230: remaining time: 22740.3 | est. epoch loss: 0.4997\n",
            "230: est. validation accuracy: 0.7115\n",
            "240: remaining time: 22726.8 | est. epoch loss: 0.4940\n",
            "240: est. validation accuracy: 0.7750\n",
            "250: remaining time: 22794.0 | est. epoch loss: 0.4878\n",
            "250: est. validation accuracy: 0.7827\n",
            "260: remaining time: 22798.4 | est. epoch loss: 0.4838\n",
            "260: est. validation accuracy: 0.7536\n",
            "270: remaining time: 22876.9 | est. epoch loss: 0.4811\n",
            "270: est. validation accuracy: 0.7788\n",
            "280: remaining time: 22758.6 | est. epoch loss: 0.4750\n",
            "280: est. validation accuracy: 0.7627\n",
            "290: remaining time: 22765.7 | est. epoch loss: 0.4728\n",
            "290: est. validation accuracy: 0.7739\n",
            "300: remaining time: 22466.8 | est. epoch loss: 0.4745\n",
            "300: est. validation accuracy: 0.7321\n",
            "310: remaining time: 22182.7 | est. epoch loss: 0.4728\n",
            "310: est. validation accuracy: 0.7714\n",
            "320: remaining time: 22307.0 | est. epoch loss: 0.4674\n",
            "320: est. validation accuracy: 0.7819\n",
            "330: remaining time: 22309.2 | est. epoch loss: 0.4621\n",
            "330: est. validation accuracy: 0.7665\n",
            "340: remaining time: 22313.7 | est. epoch loss: 0.4586\n",
            "340: est. validation accuracy: 0.7686\n",
            "350: remaining time: 22231.0 | est. epoch loss: 0.4559\n",
            "350: est. validation accuracy: 0.7748\n",
            "360: remaining time: 22378.3 | est. epoch loss: 0.4522\n",
            "360: est. validation accuracy: 0.7813\n",
            "370: remaining time: 22319.4 | est. epoch loss: 0.4476\n",
            "370: est. validation accuracy: 0.7871\n",
            "380: remaining time: 22367.8 | est. epoch loss: 0.4430\n",
            "380: est. validation accuracy: 0.7977\n",
            "390: remaining time: 22228.0 | est. epoch loss: 0.4416\n",
            "390: est. validation accuracy: 0.8002\n",
            "400: remaining time: 22255.4 | est. epoch loss: 0.4382\n",
            "400: est. validation accuracy: 0.7959\n",
            "410: remaining time: 22327.7 | est. epoch loss: 0.4349\n",
            "410: est. validation accuracy: 0.7970\n",
            "420: remaining time: 22283.1 | est. epoch loss: 0.4325\n",
            "420: est. validation accuracy: 0.7955\n",
            "430: remaining time: 22230.4 | est. epoch loss: 0.4296\n",
            "430: est. validation accuracy: 0.7971\n",
            "440: remaining time: 22139.5 | est. epoch loss: 0.4272\n",
            "440: est. validation accuracy: 0.7987\n",
            "450: remaining time: 22066.0 | est. epoch loss: 0.4253\n",
            "450: est. validation accuracy: 0.8027\n",
            "460: remaining time: 22202.4 | est. epoch loss: 0.4220\n",
            "460: est. validation accuracy: 0.8121\n",
            "470: remaining time: 22127.9 | est. epoch loss: 0.4194\n",
            "470: est. validation accuracy: 0.8039\n",
            "480: remaining time: 22010.1 | est. epoch loss: 0.4177\n",
            "480: est. validation accuracy: 0.8037\n",
            "490: remaining time: 21970.7 | est. epoch loss: 0.4156\n",
            "490: est. validation accuracy: 0.8180\n",
            "500: remaining time: 21952.0 | est. epoch loss: 0.4131\n",
            "500: est. validation accuracy: 0.8139\n",
            "510: remaining time: 21979.6 | est. epoch loss: 0.4106\n",
            "510: est. validation accuracy: 0.8034\n",
            "520: remaining time: 22008.0 | est. epoch loss: 0.4078\n",
            "520: est. validation accuracy: 0.7928\n",
            "530: remaining time: 21985.0 | est. epoch loss: 0.4049\n",
            "530: est. validation accuracy: 0.7850\n",
            "540: remaining time: 21939.0 | est. epoch loss: 0.4021\n",
            "540: est. validation accuracy: 0.7955\n",
            "550: remaining time: 22038.0 | est. epoch loss: 0.3995\n",
            "550: est. validation accuracy: 0.7933\n",
            "560: remaining time: 22082.8 | est. epoch loss: 0.3970\n",
            "560: est. validation accuracy: 0.7713\n",
            "570: remaining time: 22049.3 | est. epoch loss: 0.3954\n",
            "570: est. validation accuracy: 0.8032\n",
            "580: remaining time: 22001.1 | est. epoch loss: 0.3936\n",
            "580: est. validation accuracy: 0.7948\n",
            "590: remaining time: 21947.2 | est. epoch loss: 0.3921\n",
            "590: est. validation accuracy: 0.7928\n",
            "600: remaining time: 21855.5 | est. epoch loss: 0.3911\n",
            "600: est. validation accuracy: 0.8087\n",
            "610: remaining time: 21826.2 | est. epoch loss: 0.3887\n",
            "610: est. validation accuracy: 0.8123\n",
            "620: remaining time: 21852.0 | est. epoch loss: 0.3859\n",
            "620: est. validation accuracy: 0.8112\n",
            "630: remaining time: 21877.7 | est. epoch loss: 0.3835\n",
            "630: est. validation accuracy: 0.8118\n",
            "640: remaining time: 21958.4 | est. epoch loss: 0.3812\n",
            "640: est. validation accuracy: 0.8030\n",
            "650: remaining time: 21932.9 | est. epoch loss: 0.3795\n",
            "650: est. validation accuracy: 0.8048\n",
            "660: remaining time: 21902.9 | est. epoch loss: 0.3780\n",
            "660: est. validation accuracy: 0.8077\n",
            "670: remaining time: 21879.4 | est. epoch loss: 0.3766\n",
            "670: est. validation accuracy: 0.8095\n",
            "680: remaining time: 21913.3 | est. epoch loss: 0.3746\n",
            "680: est. validation accuracy: 0.8141\n",
            "690: remaining time: 21900.2 | est. epoch loss: 0.3725\n",
            "690: est. validation accuracy: 0.8121\n",
            "700: remaining time: 21806.7 | est. epoch loss: 0.3716\n",
            "700: est. validation accuracy: 0.8114\n",
            "710: remaining time: 21802.9 | est. epoch loss: 0.3698\n",
            "710: est. validation accuracy: 0.8059\n",
            "720: remaining time: 21748.0 | est. epoch loss: 0.3687\n",
            "720: est. validation accuracy: 0.8184\n",
            "730: remaining time: 21678.2 | est. epoch loss: 0.3674\n",
            "730: est. validation accuracy: 0.8004\n",
            "740: remaining time: 21627.3 | est. epoch loss: 0.3660\n",
            "740: est. validation accuracy: 0.8218\n",
            "750: remaining time: 21488.9 | est. epoch loss: 0.3655\n",
            "750: est. validation accuracy: 0.8261\n",
            "760: remaining time: 21455.9 | est. epoch loss: 0.3647\n",
            "760: est. validation accuracy: 0.8195\n",
            "770: remaining time: 21348.7 | est. epoch loss: 0.3642\n",
            "770: est. validation accuracy: 0.7984\n",
            "780: remaining time: 21390.7 | est. epoch loss: 0.3624\n",
            "780: est. validation accuracy: 0.8071\n",
            "790: remaining time: 21357.5 | est. epoch loss: 0.3612\n",
            "790: est. validation accuracy: 0.8171\n",
            "800: remaining time: 21301.1 | est. epoch loss: 0.3602\n",
            "800: est. validation accuracy: 0.8002\n",
            "810: remaining time: 21245.7 | est. epoch loss: 0.3591\n",
            "810: est. validation accuracy: 0.8146\n",
            "820: remaining time: 21177.4 | est. epoch loss: 0.3583\n",
            "820: est. validation accuracy: 0.7957\n",
            "830: remaining time: 21091.6 | est. epoch loss: 0.3566\n",
            "830: est. validation accuracy: 0.8164\n",
            "840: remaining time: 21063.5 | est. epoch loss: 0.3548\n",
            "840: est. validation accuracy: 0.7924\n",
            "850: remaining time: 21017.5 | est. epoch loss: 0.3538\n",
            "850: est. validation accuracy: 0.8127\n",
            "860: remaining time: 20927.2 | est. epoch loss: 0.3533\n",
            "860: est. validation accuracy: 0.8320\n",
            "870: remaining time: 20866.6 | est. epoch loss: 0.3523\n",
            "870: est. validation accuracy: 0.8179\n",
            "880: remaining time: 20907.1 | est. epoch loss: 0.3503\n",
            "880: est. validation accuracy: 0.8338\n",
            "890: remaining time: 20966.9 | est. epoch loss: 0.3495\n",
            "890: est. validation accuracy: 0.8323\n",
            "900: remaining time: 20913.6 | est. epoch loss: 0.3488\n",
            "900: est. validation accuracy: 0.8329\n",
            "910: remaining time: 20908.6 | est. epoch loss: 0.3472\n",
            "910: est. validation accuracy: 0.8277\n",
            "920: remaining time: 20894.7 | est. epoch loss: 0.3460\n",
            "920: est. validation accuracy: 0.8087\n",
            "930: remaining time: 20860.3 | est. epoch loss: 0.3455\n",
            "930: est. validation accuracy: 0.8223\n",
            "940: remaining time: 20792.9 | est. epoch loss: 0.3446\n",
            "940: est. validation accuracy: 0.8288\n",
            "950: remaining time: 20751.2 | est. epoch loss: 0.3436\n",
            "950: est. validation accuracy: 0.8257\n",
            "960: remaining time: 20724.7 | est. epoch loss: 0.3422\n",
            "960: est. validation accuracy: 0.8309\n",
            "970: remaining time: 20718.1 | est. epoch loss: 0.3410\n",
            "970: est. validation accuracy: 0.8375\n",
            "980: remaining time: 20698.5 | est. epoch loss: 0.3395\n",
            "980: est. validation accuracy: 0.8127\n",
            "990: remaining time: 20716.1 | est. epoch loss: 0.3383\n",
            "990: est. validation accuracy: 0.8195\n",
            "1000: remaining time: 20717.7 | est. epoch loss: 0.3370\n",
            "1000: est. validation accuracy: 0.8297\n",
            "1010: remaining time: 20697.1 | est. epoch loss: 0.3356\n",
            "1010: est. validation accuracy: 0.8282\n",
            "1020: remaining time: 20689.7 | est. epoch loss: 0.3341\n",
            "1020: est. validation accuracy: 0.8393\n",
            "1030: remaining time: 20673.1 | est. epoch loss: 0.3330\n",
            "1030: est. validation accuracy: 0.8343\n",
            "1040: remaining time: 20669.5 | est. epoch loss: 0.3314\n",
            "1040: est. validation accuracy: 0.8279\n",
            "1050: remaining time: 20634.6 | est. epoch loss: 0.3303\n",
            "1050: est. validation accuracy: 0.8335\n",
            "1060: remaining time: 20640.3 | est. epoch loss: 0.3287\n",
            "1060: est. validation accuracy: 0.8257\n",
            "1070: remaining time: 20625.3 | est. epoch loss: 0.3280\n",
            "1070: est. validation accuracy: 0.8171\n",
            "1080: remaining time: 20562.1 | est. epoch loss: 0.3272\n",
            "1080: est. validation accuracy: 0.8216\n",
            "1090: remaining time: 20522.1 | est. epoch loss: 0.3264\n",
            "1090: est. validation accuracy: 0.8307\n",
            "1100: remaining time: 20532.9 | est. epoch loss: 0.3253\n",
            "1100: est. validation accuracy: 0.8264\n",
            "1110: remaining time: 20548.8 | est. epoch loss: 0.3244\n",
            "1110: est. validation accuracy: 0.8071\n",
            "1120: remaining time: 20496.8 | est. epoch loss: 0.3236\n",
            "1120: est. validation accuracy: 0.8196\n",
            "1130: remaining time: 20513.2 | est. epoch loss: 0.3222\n",
            "1130: est. validation accuracy: 0.8241\n",
            "1140: remaining time: 20524.6 | est. epoch loss: 0.3210\n",
            "1140: est. validation accuracy: 0.8366\n",
            "1150: remaining time: 20458.8 | est. epoch loss: 0.3205\n",
            "1150: est. validation accuracy: 0.8290\n",
            "1160: remaining time: 20408.1 | est. epoch loss: 0.3197\n",
            "1160: est. validation accuracy: 0.8313\n",
            "1170: remaining time: 20416.4 | est. epoch loss: 0.3185\n",
            "1170: est. validation accuracy: 0.8118\n",
            "1180: remaining time: 20409.3 | est. epoch loss: 0.3178\n",
            "1180: est. validation accuracy: 0.8080\n",
            "1190: remaining time: 20341.0 | est. epoch loss: 0.3171\n",
            "1190: est. validation accuracy: 0.8358\n",
            "1200: remaining time: 20309.0 | est. epoch loss: 0.3163\n",
            "1200: est. validation accuracy: 0.8218\n",
            "1210: remaining time: 20281.3 | est. epoch loss: 0.3156\n",
            "1210: est. validation accuracy: 0.8293\n",
            "1220: remaining time: 20250.9 | est. epoch loss: 0.3145\n",
            "1220: est. validation accuracy: 0.8318\n",
            "1230: remaining time: 20281.3 | est. epoch loss: 0.3130\n",
            "1230: est. validation accuracy: 0.8296\n",
            "1240: remaining time: 20203.6 | est. epoch loss: 0.3130\n",
            "1240: est. validation accuracy: 0.8275\n",
            "1250: remaining time: 20153.0 | est. epoch loss: 0.3124\n",
            "1250: est. validation accuracy: 0.8116\n",
            "1260: remaining time: 20146.8 | est. epoch loss: 0.3114\n",
            "1260: est. validation accuracy: 0.8132\n",
            "1270: remaining time: 20149.6 | est. epoch loss: 0.3103\n",
            "1270: est. validation accuracy: 0.8193\n",
            "1280: remaining time: 20124.0 | est. epoch loss: 0.3096\n",
            "1280: est. validation accuracy: 0.8363\n",
            "1290: remaining time: 20055.3 | est. epoch loss: 0.3092\n",
            "1290: est. validation accuracy: 0.8305\n",
            "1300: remaining time: 20008.3 | est. epoch loss: 0.3087\n",
            "1300: est. validation accuracy: 0.8357\n",
            "1310: remaining time: 20019.2 | est. epoch loss: 0.3072\n",
            "1310: est. validation accuracy: 0.8376\n",
            "1320: remaining time: 20012.8 | est. epoch loss: 0.3059\n",
            "1320: est. validation accuracy: 0.8130\n",
            "1330: remaining time: 19976.7 | est. epoch loss: 0.3050\n",
            "1330: est. validation accuracy: 0.8350\n",
            "1340: remaining time: 19940.6 | est. epoch loss: 0.3041\n",
            "1340: est. validation accuracy: 0.8477\n",
            "1350: remaining time: 19880.2 | est. epoch loss: 0.3037\n",
            "1350: est. validation accuracy: 0.7673\n",
            "1360: remaining time: 19846.1 | est. epoch loss: 0.3028\n",
            "1360: est. validation accuracy: 0.8435\n",
            "1370: remaining time: 19823.8 | est. epoch loss: 0.3020\n",
            "1370: est. validation accuracy: 0.8418\n",
            "1380: remaining time: 19782.1 | est. epoch loss: 0.3014\n",
            "1380: est. validation accuracy: 0.8071\n",
            "1390: remaining time: 19751.9 | est. epoch loss: 0.3006\n",
            "1390: est. validation accuracy: 0.8243\n",
            "1400: remaining time: 19744.2 | est. epoch loss: 0.2995\n",
            "1400: est. validation accuracy: 0.8175\n",
            "1410: remaining time: 19738.5 | est. epoch loss: 0.2987\n",
            "1410: est. validation accuracy: 0.8286\n",
            "1420: remaining time: 19728.6 | est. epoch loss: 0.2977\n",
            "1420: est. validation accuracy: 0.8064\n",
            "1430: remaining time: 19726.1 | est. epoch loss: 0.2969\n",
            "1430: est. validation accuracy: 0.8407\n",
            "1440: remaining time: 19712.3 | est. epoch loss: 0.2959\n",
            "1440: est. validation accuracy: 0.8357\n",
            "1450: remaining time: 19683.7 | est. epoch loss: 0.2952\n",
            "1450: est. validation accuracy: 0.8350\n",
            "1460: remaining time: 19673.1 | est. epoch loss: 0.2939\n",
            "1460: est. validation accuracy: 0.8323\n",
            "1470: remaining time: 19666.7 | est. epoch loss: 0.2931\n",
            "1470: est. validation accuracy: 0.8272\n",
            "1480: remaining time: 19669.7 | est. epoch loss: 0.2922\n",
            "1480: est. validation accuracy: 0.8304\n",
            "1490: remaining time: 19676.6 | est. epoch loss: 0.2912\n",
            "1490: est. validation accuracy: 0.8148\n",
            "1500: remaining time: 19634.2 | est. epoch loss: 0.2906\n",
            "1500: est. validation accuracy: 0.8243\n",
            "1510: remaining time: 19608.4 | est. epoch loss: 0.2896\n",
            "1510: est. validation accuracy: 0.8200\n",
            "1520: remaining time: 19583.4 | est. epoch loss: 0.2889\n",
            "1520: est. validation accuracy: 0.8112\n",
            "1530: remaining time: 19509.5 | est. epoch loss: 0.2891\n",
            "1530: est. validation accuracy: 0.8396\n",
            "1540: remaining time: 19480.5 | est. epoch loss: 0.2883\n",
            "1540: est. validation accuracy: 0.8373\n",
            "1550: remaining time: 19431.7 | est. epoch loss: 0.2877\n",
            "1550: est. validation accuracy: 0.8327\n",
            "1560: remaining time: 19396.0 | est. epoch loss: 0.2871\n",
            "1560: est. validation accuracy: 0.8281\n",
            "1570: remaining time: 19400.1 | est. epoch loss: 0.2862\n",
            "1570: est. validation accuracy: 0.8222\n",
            "1580: remaining time: 19364.1 | est. epoch loss: 0.2855\n",
            "1580: est. validation accuracy: 0.8353\n",
            "1590: remaining time: 19315.0 | est. epoch loss: 0.2850\n",
            "1590: est. validation accuracy: 0.8024\n",
            "1600: remaining time: 19244.7 | est. epoch loss: 0.2847\n",
            "1600: est. validation accuracy: 0.8267\n",
            "1610: remaining time: 19249.8 | est. epoch loss: 0.2836\n",
            "1610: est. validation accuracy: 0.8333\n",
            "1620: remaining time: 19199.0 | est. epoch loss: 0.2831\n",
            "1620: est. validation accuracy: 0.7828\n",
            "1630: remaining time: 19185.0 | est. epoch loss: 0.2824\n",
            "1630: est. validation accuracy: 0.8404\n",
            "1640: remaining time: 19113.3 | est. epoch loss: 0.2827\n",
            "1640: est. validation accuracy: 0.8315\n",
            "1650: remaining time: 19042.3 | est. epoch loss: 0.2824\n",
            "1650: est. validation accuracy: 0.8062\n",
            "1660: remaining time: 19018.6 | est. epoch loss: 0.2818\n",
            "1660: est. validation accuracy: 0.8512\n",
            "1670: remaining time: 18983.2 | est. epoch loss: 0.2812\n",
            "1670: est. validation accuracy: 0.8168\n",
            "1680: remaining time: 18983.9 | est. epoch loss: 0.2802\n",
            "1680: est. validation accuracy: 0.8451\n",
            "1690: remaining time: 18973.6 | est. epoch loss: 0.2794\n",
            "1690: est. validation accuracy: 0.8456\n",
            "1700: remaining time: 18925.8 | est. epoch loss: 0.2793\n",
            "1700: est. validation accuracy: 0.8552\n",
            "1710: remaining time: 18883.4 | est. epoch loss: 0.2786\n",
            "1710: est. validation accuracy: 0.8177\n",
            "1720: remaining time: 18856.4 | est. epoch loss: 0.2776\n",
            "1720: est. validation accuracy: 0.7994\n",
            "1730: remaining time: 18819.0 | est. epoch loss: 0.2772\n",
            "1730: est. validation accuracy: 0.7903\n",
            "1740: remaining time: 18800.3 | est. epoch loss: 0.2764\n",
            "1740: est. validation accuracy: 0.8121\n",
            "1750: remaining time: 18775.3 | est. epoch loss: 0.2758\n",
            "1750: est. validation accuracy: 0.7883\n",
            "1760: remaining time: 18751.3 | est. epoch loss: 0.2753\n",
            "1760: est. validation accuracy: 0.8121\n",
            "1770: remaining time: 18713.6 | est. epoch loss: 0.2748\n",
            "1770: est. validation accuracy: 0.8069\n",
            "1780: remaining time: 18685.3 | est. epoch loss: 0.2743\n",
            "1780: est. validation accuracy: 0.8161\n",
            "1790: remaining time: 18644.8 | est. epoch loss: 0.2739\n",
            "1790: est. validation accuracy: 0.8352\n",
            "1800: remaining time: 18602.1 | est. epoch loss: 0.2736\n",
            "1800: est. validation accuracy: 0.8365\n",
            "1810: remaining time: 18573.7 | est. epoch loss: 0.2730\n",
            "1810: est. validation accuracy: 0.8374\n",
            "1820: remaining time: 18554.3 | est. epoch loss: 0.2723\n",
            "1820: est. validation accuracy: 0.8449\n",
            "1830: remaining time: 18517.1 | est. epoch loss: 0.2716\n",
            "1830: est. validation accuracy: 0.8474\n",
            "1840: remaining time: 18529.8 | est. epoch loss: 0.2707\n",
            "1840: est. validation accuracy: 0.7989\n",
            "1850: remaining time: 18514.0 | est. epoch loss: 0.2700\n",
            "1850: est. validation accuracy: 0.8293\n",
            "1860: remaining time: 18485.3 | est. epoch loss: 0.2693\n",
            "1860: est. validation accuracy: 0.8241\n",
            "1870: remaining time: 18439.3 | est. epoch loss: 0.2690\n",
            "1870: est. validation accuracy: 0.8111\n",
            "1880: remaining time: 18390.7 | est. epoch loss: 0.2687\n",
            "1880: est. validation accuracy: 0.8412\n",
            "1890: remaining time: 18343.2 | est. epoch loss: 0.2687\n",
            "1890: est. validation accuracy: 0.8152\n",
            "1900: remaining time: 18326.5 | est. epoch loss: 0.2680\n",
            "1900: est. validation accuracy: 0.8370\n",
            "1910: remaining time: 18293.7 | est. epoch loss: 0.2674\n",
            "1910: est. validation accuracy: 0.8476\n",
            "1920: remaining time: 18278.7 | est. epoch loss: 0.2665\n",
            "1920: est. validation accuracy: 0.8209\n",
            "1930: remaining time: 18295.7 | est. epoch loss: 0.2659\n",
            "1930: est. validation accuracy: 0.8286\n",
            "1940: remaining time: 18234.8 | est. epoch loss: 0.2658\n",
            "1940: est. validation accuracy: 0.8145\n",
            "1950: remaining time: 18219.3 | est. epoch loss: 0.2651\n",
            "1950: est. validation accuracy: 0.8356\n",
            "1960: remaining time: 18202.6 | est. epoch loss: 0.2646\n",
            "1960: est. validation accuracy: 0.8520\n",
            "1970: remaining time: 18185.1 | est. epoch loss: 0.2642\n",
            "1970: est. validation accuracy: 0.8318\n",
            "1980: remaining time: 18152.0 | est. epoch loss: 0.2638\n",
            "1980: est. validation accuracy: 0.8474\n",
            "1990: remaining time: 18155.4 | est. epoch loss: 0.2630\n",
            "1990: est. validation accuracy: 0.8404\n",
            "2000: remaining time: 18122.8 | est. epoch loss: 0.2624\n",
            "2000: est. validation accuracy: 0.8373\n",
            "2010: remaining time: 18114.7 | est. epoch loss: 0.2617\n",
            "2010: est. validation accuracy: 0.8369\n",
            "2020: remaining time: 18084.7 | est. epoch loss: 0.2612\n",
            "2020: est. validation accuracy: 0.8395\n",
            "2030: remaining time: 18047.4 | est. epoch loss: 0.2607\n",
            "2030: est. validation accuracy: 0.8168\n",
            "2040: remaining time: 18019.2 | est. epoch loss: 0.2605\n",
            "2040: est. validation accuracy: 0.8413\n",
            "2050: remaining time: 17982.9 | est. epoch loss: 0.2598\n",
            "2050: est. validation accuracy: 0.8517\n",
            "2060: remaining time: 17935.3 | est. epoch loss: 0.2596\n",
            "2060: est. validation accuracy: 0.8490\n",
            "2070: remaining time: 17916.4 | est. epoch loss: 0.2591\n",
            "2070: est. validation accuracy: 0.8225\n",
            "2080: remaining time: 17884.9 | est. epoch loss: 0.2588\n",
            "2080: est. validation accuracy: 0.8245\n",
            "2090: remaining time: 17848.7 | est. epoch loss: 0.2586\n",
            "2090: est. validation accuracy: 0.8426\n",
            "2100: remaining time: 17789.7 | est. epoch loss: 0.2585\n",
            "2100: est. validation accuracy: 0.8436\n",
            "2110: remaining time: 17785.7 | est. epoch loss: 0.2578\n",
            "2110: est. validation accuracy: 0.8306\n",
            "2120: remaining time: 17758.4 | est. epoch loss: 0.2573\n",
            "2120: est. validation accuracy: 0.7845\n",
            "2130: remaining time: 17705.8 | est. epoch loss: 0.2573\n",
            "2130: est. validation accuracy: 0.8394\n",
            "2140: remaining time: 17694.4 | est. epoch loss: 0.2567\n",
            "2140: est. validation accuracy: 0.8354\n",
            "2150: remaining time: 17663.2 | est. epoch loss: 0.2564\n",
            "2150: est. validation accuracy: 0.7903\n",
            "2160: remaining time: 17628.1 | est. epoch loss: 0.2560\n",
            "2160: est. validation accuracy: 0.8326\n",
            "2170: remaining time: 17614.0 | est. epoch loss: 0.2550\n",
            "2170: est. validation accuracy: 0.8261\n",
            "2180: remaining time: 17567.1 | est. epoch loss: 0.2548\n",
            "2180: est. validation accuracy: 0.8306\n",
            "2190: remaining time: 17529.3 | est. epoch loss: 0.2546\n",
            "2190: est. validation accuracy: 0.8286\n",
            "2200: remaining time: 17511.7 | est. epoch loss: 0.2542\n",
            "2200: est. validation accuracy: 0.8200\n",
            "2210: remaining time: 17495.1 | est. epoch loss: 0.2536\n",
            "2210: est. validation accuracy: 0.8410\n",
            "2220: remaining time: 17473.3 | est. epoch loss: 0.2530\n",
            "2220: est. validation accuracy: 0.8404\n",
            "2230: remaining time: 17468.6 | est. epoch loss: 0.2521\n",
            "2230: est. validation accuracy: 0.8295\n",
            "2240: remaining time: 17429.0 | est. epoch loss: 0.2520\n",
            "2240: est. validation accuracy: 0.8441\n",
            "2250: remaining time: 17401.2 | est. epoch loss: 0.2515\n",
            "2250: est. validation accuracy: 0.8430\n",
            "2260: remaining time: 17398.2 | est. epoch loss: 0.2508\n",
            "2260: est. validation accuracy: 0.8448\n",
            "2270: remaining time: 17359.7 | est. epoch loss: 0.2506\n",
            "2270: est. validation accuracy: 0.8087\n",
            "2280: remaining time: 17320.7 | est. epoch loss: 0.2502\n",
            "2280: est. validation accuracy: 0.8245\n",
            "2290: remaining time: 17292.6 | est. epoch loss: 0.2501\n",
            "2290: est. validation accuracy: 0.8250\n",
            "2300: remaining time: 17259.9 | est. epoch loss: 0.2495\n",
            "2300: est. validation accuracy: 0.8191\n",
            "2310: remaining time: 17231.5 | est. epoch loss: 0.2490\n",
            "2310: est. validation accuracy: 0.8508\n",
            "2320: remaining time: 17199.5 | est. epoch loss: 0.2486\n",
            "2320: est. validation accuracy: 0.8100\n",
            "2330: remaining time: 17192.8 | est. epoch loss: 0.2479\n",
            "2330: est. validation accuracy: 0.8521\n",
            "2340: remaining time: 17185.1 | est. epoch loss: 0.2473\n",
            "2340: est. validation accuracy: 0.8267\n",
            "2350: remaining time: 17149.6 | est. epoch loss: 0.2472\n",
            "2350: est. validation accuracy: 0.8138\n",
            "2360: remaining time: 17110.1 | est. epoch loss: 0.2469\n",
            "2360: est. validation accuracy: 0.8324\n",
            "2370: remaining time: 17064.9 | est. epoch loss: 0.2466\n",
            "2370: est. validation accuracy: 0.8284\n",
            "2380: remaining time: 17029.1 | est. epoch loss: 0.2464\n",
            "2380: est. validation accuracy: 0.8250\n",
            "2390: remaining time: 16999.9 | est. epoch loss: 0.2461\n",
            "2390: est. validation accuracy: 0.8293\n",
            "2400: remaining time: 16968.3 | est. epoch loss: 0.2458\n",
            "2400: est. validation accuracy: 0.8130\n",
            "2410: remaining time: 16929.1 | est. epoch loss: 0.2456\n",
            "2410: est. validation accuracy: 0.8223\n",
            "2420: remaining time: 16896.6 | est. epoch loss: 0.2450\n",
            "2420: est. validation accuracy: 0.7654\n",
            "2430: remaining time: 16871.8 | est. epoch loss: 0.2447\n",
            "2430: est. validation accuracy: 0.8295\n",
            "2440: remaining time: 16854.1 | est. epoch loss: 0.2441\n",
            "2440: est. validation accuracy: 0.8270\n",
            "2450: remaining time: 16839.6 | est. epoch loss: 0.2435\n",
            "2450: est. validation accuracy: 0.8182\n",
            "2460: remaining time: 16820.9 | est. epoch loss: 0.2431\n",
            "2460: est. validation accuracy: 0.8245\n",
            "2470: remaining time: 16786.8 | est. epoch loss: 0.2428\n",
            "2470: est. validation accuracy: 0.8107\n",
            "2480: remaining time: 16748.9 | est. epoch loss: 0.2425\n",
            "2480: est. validation accuracy: 0.8369\n",
            "2490: remaining time: 16734.6 | est. epoch loss: 0.2420\n",
            "2490: est. validation accuracy: 0.8346\n",
            "2500: remaining time: 16702.9 | est. epoch loss: 0.2417\n",
            "2500: est. validation accuracy: 0.8114\n",
            "2510: remaining time: 16695.2 | est. epoch loss: 0.2411\n",
            "2510: est. validation accuracy: 0.8470\n",
            "2520: remaining time: 16680.3 | est. epoch loss: 0.2404\n",
            "2520: est. validation accuracy: 0.8549\n",
            "2530: remaining time: 16671.2 | est. epoch loss: 0.2398\n",
            "2530: est. validation accuracy: 0.7831\n",
            "2540: remaining time: 16644.6 | est. epoch loss: 0.2395\n",
            "2540: est. validation accuracy: 0.8420\n",
            "2550: remaining time: 16635.2 | est. epoch loss: 0.2387\n",
            "2550: est. validation accuracy: 0.8446\n",
            "2560: remaining time: 16618.5 | est. epoch loss: 0.2381\n",
            "2560: est. validation accuracy: 0.8181\n",
            "2570: remaining time: 16596.1 | est. epoch loss: 0.2379\n",
            "2570: est. validation accuracy: 0.8274\n",
            "2580: remaining time: 16581.9 | est. epoch loss: 0.2373\n",
            "2580: est. validation accuracy: 0.8361\n",
            "2590: remaining time: 16564.1 | est. epoch loss: 0.2369\n",
            "2590: est. validation accuracy: 0.8145\n",
            "2600: remaining time: 16529.0 | est. epoch loss: 0.2365\n",
            "2600: est. validation accuracy: 0.8271\n",
            "2610: remaining time: 16496.4 | est. epoch loss: 0.2362\n",
            "2610: est. validation accuracy: 0.8254\n",
            "2620: remaining time: 16485.0 | est. epoch loss: 0.2357\n",
            "2620: est. validation accuracy: 0.8386\n",
            "2630: remaining time: 16442.6 | est. epoch loss: 0.2356\n",
            "2630: est. validation accuracy: 0.8421\n",
            "2640: remaining time: 16418.5 | est. epoch loss: 0.2353\n",
            "2640: est. validation accuracy: 0.7948\n",
            "2650: remaining time: 16390.0 | est. epoch loss: 0.2349\n",
            "2650: est. validation accuracy: 0.8412\n",
            "2660: remaining time: 16346.1 | est. epoch loss: 0.2349\n",
            "2660: est. validation accuracy: 0.7701\n",
            "2670: remaining time: 16321.8 | est. epoch loss: 0.2346\n",
            "2670: est. validation accuracy: 0.8345\n",
            "2680: remaining time: 16303.8 | est. epoch loss: 0.2342\n",
            "2680: est. validation accuracy: 0.7871\n",
            "2690: remaining time: 16292.4 | est. epoch loss: 0.2337\n",
            "2690: est. validation accuracy: 0.8181\n",
            "2700: remaining time: 16271.6 | est. epoch loss: 0.2333\n",
            "2700: est. validation accuracy: 0.8277\n",
            "2710: remaining time: 16249.4 | est. epoch loss: 0.2329\n",
            "2710: est. validation accuracy: 0.8105\n",
            "2720: remaining time: 16221.6 | est. epoch loss: 0.2325\n",
            "2720: est. validation accuracy: 0.8420\n",
            "2730: remaining time: 16201.7 | est. epoch loss: 0.2321\n",
            "2730: est. validation accuracy: 0.8000\n",
            "2740: remaining time: 16165.9 | est. epoch loss: 0.2320\n",
            "2740: est. validation accuracy: 0.8284\n",
            "2750: remaining time: 16141.8 | est. epoch loss: 0.2315\n",
            "2750: est. validation accuracy: 0.8314\n",
            "2760: remaining time: 16128.4 | est. epoch loss: 0.2311\n",
            "2760: est. validation accuracy: 0.8358\n",
            "2770: remaining time: 16119.0 | est. epoch loss: 0.2305\n",
            "2770: est. validation accuracy: 0.8384\n",
            "2780: remaining time: 16098.3 | est. epoch loss: 0.2301\n",
            "2780: est. validation accuracy: 0.8336\n",
            "2790: remaining time: 16056.9 | est. epoch loss: 0.2299\n",
            "2790: est. validation accuracy: 0.8218\n",
            "2800: remaining time: 16044.5 | est. epoch loss: 0.2294\n",
            "2800: est. validation accuracy: 0.8425\n",
            "2810: remaining time: 16017.3 | est. epoch loss: 0.2290\n",
            "2810: est. validation accuracy: 0.8155\n",
            "2820: remaining time: 15997.4 | est. epoch loss: 0.2286\n",
            "2820: est. validation accuracy: 0.8091\n",
            "2830: remaining time: 15991.6 | est. epoch loss: 0.2281\n",
            "2830: est. validation accuracy: 0.8070\n",
            "2840: remaining time: 15961.7 | est. epoch loss: 0.2276\n",
            "2840: est. validation accuracy: 0.8299\n",
            "2850: remaining time: 15930.8 | est. epoch loss: 0.2273\n",
            "2850: est. validation accuracy: 0.8433\n",
            "2860: remaining time: 15894.5 | est. epoch loss: 0.2270\n",
            "2860: est. validation accuracy: 0.8171\n",
            "2870: remaining time: 15865.2 | est. epoch loss: 0.2267\n",
            "2870: est. validation accuracy: 0.8302\n",
            "2880: remaining time: 15827.9 | est. epoch loss: 0.2265\n",
            "2880: est. validation accuracy: 0.8453\n",
            "2890: remaining time: 15795.5 | est. epoch loss: 0.2262\n",
            "2890: est. validation accuracy: 0.8421\n",
            "2900: remaining time: 15774.2 | est. epoch loss: 0.2258\n",
            "2900: est. validation accuracy: 0.8032\n",
            "2910: remaining time: 15749.8 | est. epoch loss: 0.2254\n",
            "2910: est. validation accuracy: 0.8552\n",
            "2920: remaining time: 15718.0 | est. epoch loss: 0.2251\n",
            "2920: est. validation accuracy: 0.8218\n",
            "2930: remaining time: 15687.8 | est. epoch loss: 0.2247\n",
            "2930: est. validation accuracy: 0.8394\n",
            "2940: remaining time: 15671.0 | est. epoch loss: 0.2243\n",
            "2940: est. validation accuracy: 0.8465\n",
            "2950: remaining time: 15647.0 | est. epoch loss: 0.2238\n",
            "2950: est. validation accuracy: 0.8322\n",
            "2960: remaining time: 15637.7 | est. epoch loss: 0.2232\n",
            "2960: est. validation accuracy: 0.8454\n",
            "2970: remaining time: 15615.3 | est. epoch loss: 0.2229\n",
            "2970: est. validation accuracy: 0.8037\n",
            "2980: remaining time: 15582.0 | est. epoch loss: 0.2228\n",
            "2980: est. validation accuracy: 0.8441\n",
            "2990: remaining time: 15548.1 | est. epoch loss: 0.2225\n",
            "2990: est. validation accuracy: 0.8227\n",
            "3000: remaining time: 15520.7 | est. epoch loss: 0.2222\n",
            "3000: est. validation accuracy: 0.8087\n",
            "3010: remaining time: 15508.8 | est. epoch loss: 0.2218\n",
            "3010: est. validation accuracy: 0.8240\n",
            "3020: remaining time: 15477.9 | est. epoch loss: 0.2215\n",
            "3020: est. validation accuracy: 0.8274\n",
            "3030: remaining time: 15450.4 | est. epoch loss: 0.2212\n",
            "3030: est. validation accuracy: 0.8170\n",
            "3040: remaining time: 15427.5 | est. epoch loss: 0.2206\n",
            "3040: est. validation accuracy: 0.8403\n",
            "3050: remaining time: 15390.8 | est. epoch loss: 0.2205\n",
            "3050: est. validation accuracy: 0.7926\n",
            "3060: remaining time: 15374.6 | est. epoch loss: 0.2200\n",
            "3060: est. validation accuracy: 0.8435\n",
            "3070: remaining time: 15343.6 | est. epoch loss: 0.2197\n",
            "3070: est. validation accuracy: 0.8386\n",
            "3080: remaining time: 15309.8 | est. epoch loss: 0.2195\n",
            "3080: est. validation accuracy: 0.8347\n",
            "3090: remaining time: 15281.3 | est. epoch loss: 0.2192\n",
            "3090: est. validation accuracy: 0.8345\n",
            "3100: remaining time: 15261.4 | est. epoch loss: 0.2189\n",
            "3100: est. validation accuracy: 0.8424\n",
            "3110: remaining time: 15228.8 | est. epoch loss: 0.2187\n",
            "3110: est. validation accuracy: 0.7862\n",
            "3120: remaining time: 15215.7 | est. epoch loss: 0.2182\n",
            "3120: est. validation accuracy: 0.8517\n",
            "3130: remaining time: 15180.8 | est. epoch loss: 0.2181\n",
            "3130: est. validation accuracy: 0.8077\n",
            "3140: remaining time: 15140.9 | est. epoch loss: 0.2179\n",
            "3140: est. validation accuracy: 0.8141\n",
            "3150: remaining time: 15105.4 | est. epoch loss: 0.2178\n",
            "3150: est. validation accuracy: 0.8496\n",
            "3160: remaining time: 15075.6 | est. epoch loss: 0.2175\n",
            "3160: est. validation accuracy: 0.8295\n",
            "3170: remaining time: 15054.8 | est. epoch loss: 0.2170\n",
            "3170: est. validation accuracy: 0.8009\n",
            "3180: remaining time: 15037.2 | est. epoch loss: 0.2166\n",
            "3180: est. validation accuracy: 0.8105\n",
            "3190: remaining time: 15022.9 | est. epoch loss: 0.2162\n",
            "3190: est. validation accuracy: 0.8191\n",
            "3200: remaining time: 14992.7 | est. epoch loss: 0.2160\n",
            "3200: est. validation accuracy: 0.8035\n",
            "3210: remaining time: 14965.9 | est. epoch loss: 0.2156\n",
            "3210: est. validation accuracy: 0.8517\n",
            "3220: remaining time: 14932.6 | est. epoch loss: 0.2154\n",
            "3220: est. validation accuracy: 0.7968\n",
            "3230: remaining time: 14919.5 | est. epoch loss: 0.2150\n",
            "3230: est. validation accuracy: 0.7930\n",
            "3240: remaining time: 14895.6 | est. epoch loss: 0.2146\n",
            "3240: est. validation accuracy: 0.8339\n",
            "3250: remaining time: 14869.9 | est. epoch loss: 0.2142\n",
            "3250: est. validation accuracy: 0.8360\n",
            "3260: remaining time: 14830.6 | est. epoch loss: 0.2141\n",
            "3260: est. validation accuracy: 0.7942\n",
            "3270: remaining time: 14801.7 | est. epoch loss: 0.2139\n",
            "3270: est. validation accuracy: 0.7882\n",
            "3280: remaining time: 14773.7 | est. epoch loss: 0.2137\n",
            "3280: est. validation accuracy: 0.8109\n",
            "3290: remaining time: 14757.3 | est. epoch loss: 0.2133\n",
            "3290: est. validation accuracy: 0.8200\n",
            "3300: remaining time: 14741.6 | est. epoch loss: 0.2128\n",
            "3300: est. validation accuracy: 0.8309\n",
            "3310: remaining time: 14711.4 | est. epoch loss: 0.2126\n",
            "3310: est. validation accuracy: 0.8170\n",
            "3320: remaining time: 14683.6 | est. epoch loss: 0.2124\n",
            "3320: est. validation accuracy: 0.7998\n",
            "3330: remaining time: 14650.2 | est. epoch loss: 0.2123\n",
            "3330: est. validation accuracy: 0.8381\n",
            "3340: remaining time: 14626.5 | est. epoch loss: 0.2119\n",
            "3340: est. validation accuracy: 0.8455\n",
            "3350: remaining time: 14589.8 | est. epoch loss: 0.2116\n",
            "3350: est. validation accuracy: 0.8409\n",
            "3360: remaining time: 14554.2 | est. epoch loss: 0.2115\n",
            "3360: est. validation accuracy: 0.7984\n",
            "3370: remaining time: 14539.6 | est. epoch loss: 0.2110\n",
            "3370: est. validation accuracy: 0.8491\n",
            "3380: remaining time: 14512.0 | est. epoch loss: 0.2107\n",
            "3380: est. validation accuracy: 0.8423\n",
            "3390: remaining time: 14483.5 | est. epoch loss: 0.2104\n",
            "3390: est. validation accuracy: 0.8329\n",
            "3400: remaining time: 14452.2 | est. epoch loss: 0.2102\n",
            "3400: est. validation accuracy: 0.8255\n",
            "3410: remaining time: 14423.3 | est. epoch loss: 0.2100\n",
            "3410: est. validation accuracy: 0.8048\n",
            "3420: remaining time: 14386.9 | est. epoch loss: 0.2099\n",
            "3420: est. validation accuracy: 0.8225\n",
            "3430: remaining time: 14356.4 | est. epoch loss: 0.2096\n",
            "3430: est. validation accuracy: 0.8196\n",
            "3440: remaining time: 14334.7 | est. epoch loss: 0.2094\n",
            "3440: est. validation accuracy: 0.8297\n",
            "3450: remaining time: 14304.8 | est. epoch loss: 0.2092\n",
            "3450: est. validation accuracy: 0.8116\n",
            "3460: remaining time: 14286.8 | est. epoch loss: 0.2088\n",
            "3460: est. validation accuracy: 0.8005\n",
            "3470: remaining time: 14264.0 | est. epoch loss: 0.2085\n",
            "3470: est. validation accuracy: 0.8061\n",
            "3480: remaining time: 14238.9 | est. epoch loss: 0.2082\n",
            "3480: est. validation accuracy: 0.8371\n",
            "3490: remaining time: 14209.3 | est. epoch loss: 0.2079\n",
            "3490: est. validation accuracy: 0.8281\n",
            "3500: remaining time: 14195.3 | est. epoch loss: 0.2076\n",
            "3500: est. validation accuracy: 0.7816\n",
            "3510: remaining time: 14175.3 | est. epoch loss: 0.2072\n",
            "3510: est. validation accuracy: 0.8378\n",
            "3520: remaining time: 14143.2 | est. epoch loss: 0.2070\n",
            "3520: est. validation accuracy: 0.8313\n",
            "3530: remaining time: 14126.5 | est. epoch loss: 0.2067\n",
            "3530: est. validation accuracy: 0.8299\n",
            "3540: remaining time: 14090.9 | est. epoch loss: 0.2066\n",
            "3540: est. validation accuracy: 0.8407\n",
            "3550: remaining time: 14048.8 | est. epoch loss: 0.2066\n",
            "3550: est. validation accuracy: 0.8271\n",
            "3560: remaining time: 14018.8 | est. epoch loss: 0.2063\n",
            "3560: est. validation accuracy: 0.8552\n",
            "3570: remaining time: 13998.5 | est. epoch loss: 0.2061\n",
            "3570: est. validation accuracy: 0.8330\n",
            "3580: remaining time: 13965.6 | est. epoch loss: 0.2060\n",
            "3580: est. validation accuracy: 0.7785\n",
            "3590: remaining time: 13935.9 | est. epoch loss: 0.2057\n",
            "3590: est. validation accuracy: 0.8432\n",
            "3600: remaining time: 13904.9 | est. epoch loss: 0.2055\n",
            "3600: est. validation accuracy: 0.8455\n",
            "3610: remaining time: 13874.8 | est. epoch loss: 0.2052\n",
            "3610: est. validation accuracy: 0.7894\n",
            "3620: remaining time: 13852.7 | est. epoch loss: 0.2048\n",
            "3620: est. validation accuracy: 0.8091\n",
            "3630: remaining time: 13819.3 | est. epoch loss: 0.2048\n",
            "3630: est. validation accuracy: 0.8521\n",
            "3640: remaining time: 13799.4 | est. epoch loss: 0.2044\n",
            "3640: est. validation accuracy: 0.8227\n",
            "3650: remaining time: 13776.1 | est. epoch loss: 0.2041\n",
            "3650: est. validation accuracy: 0.8200\n",
            "3660: remaining time: 13759.5 | est. epoch loss: 0.2038\n",
            "3660: est. validation accuracy: 0.8050\n",
            "3670: remaining time: 13728.7 | est. epoch loss: 0.2035\n",
            "3670: est. validation accuracy: 0.8104\n",
            "3680: remaining time: 13701.4 | est. epoch loss: 0.2034\n",
            "3680: est. validation accuracy: 0.7869\n",
            "3690: remaining time: 13670.4 | est. epoch loss: 0.2032\n",
            "3690: est. validation accuracy: 0.7746\n",
            "3700: remaining time: 13640.0 | est. epoch loss: 0.2030\n",
            "3700: est. validation accuracy: 0.7935\n",
            "3710: remaining time: 13618.3 | est. epoch loss: 0.2026\n",
            "3710: est. validation accuracy: 0.8530\n",
            "3720: remaining time: 13593.3 | est. epoch loss: 0.2024\n",
            "3720: est. validation accuracy: 0.7934\n",
            "3730: remaining time: 13578.6 | est. epoch loss: 0.2020\n",
            "3730: est. validation accuracy: 0.7839\n",
            "3740: remaining time: 13546.6 | est. epoch loss: 0.2020\n",
            "3740: est. validation accuracy: 0.8114\n",
            "3750: remaining time: 13510.3 | est. epoch loss: 0.2017\n",
            "3750: est. validation accuracy: 0.8540\n",
            "3760: remaining time: 13481.3 | est. epoch loss: 0.2015\n",
            "3760: est. validation accuracy: 0.8395\n",
            "3770: remaining time: 13452.1 | est. epoch loss: 0.2012\n",
            "3770: est. validation accuracy: 0.8057\n",
            "3780: remaining time: 13429.1 | est. epoch loss: 0.2010\n",
            "3780: est. validation accuracy: 0.8484\n",
            "3790: remaining time: 13397.1 | est. epoch loss: 0.2008\n",
            "3790: est. validation accuracy: 0.8382\n",
            "3800: remaining time: 13371.2 | est. epoch loss: 0.2005\n",
            "3800: est. validation accuracy: 0.8100\n",
            "3810: remaining time: 13346.4 | est. epoch loss: 0.2002\n",
            "3810: est. validation accuracy: 0.8377\n",
            "3820: remaining time: 13316.2 | est. epoch loss: 0.1999\n",
            "3820: est. validation accuracy: 0.7971\n",
            "3830: remaining time: 13289.7 | est. epoch loss: 0.1997\n",
            "3830: est. validation accuracy: 0.8273\n",
            "3840: remaining time: 13258.8 | est. epoch loss: 0.1994\n",
            "3840: est. validation accuracy: 0.7698\n",
            "3850: remaining time: 13235.4 | est. epoch loss: 0.1994\n",
            "3850: est. validation accuracy: 0.8162\n",
            "3860: remaining time: 13208.5 | est. epoch loss: 0.1991\n",
            "3860: est. validation accuracy: 0.7707\n",
            "3870: remaining time: 13185.1 | est. epoch loss: 0.1989\n",
            "3870: est. validation accuracy: 0.8204\n",
            "3880: remaining time: 13164.0 | est. epoch loss: 0.1986\n",
            "3880: est. validation accuracy: 0.8053\n",
            "3890: remaining time: 13127.9 | est. epoch loss: 0.1985\n",
            "3890: est. validation accuracy: 0.8412\n",
            "3900: remaining time: 13092.2 | est. epoch loss: 0.1984\n",
            "3900: est. validation accuracy: 0.8598\n",
            "3910: remaining time: 13071.2 | est. epoch loss: 0.1981\n",
            "3910: est. validation accuracy: 0.8562\n",
            "3920: remaining time: 13047.2 | est. epoch loss: 0.1979\n",
            "3920: est. validation accuracy: 0.8079\n",
            "3930: remaining time: 13023.7 | est. epoch loss: 0.1977\n",
            "3930: est. validation accuracy: 0.8291\n",
            "3940: remaining time: 12997.4 | est. epoch loss: 0.1975\n",
            "3940: est. validation accuracy: 0.8436\n",
            "3950: remaining time: 12976.6 | est. epoch loss: 0.1971\n",
            "3950: est. validation accuracy: 0.8459\n",
            "3960: remaining time: 12952.8 | est. epoch loss: 0.1969\n",
            "3960: est. validation accuracy: 0.8089\n",
            "3970: remaining time: 12923.8 | est. epoch loss: 0.1967\n",
            "3970: est. validation accuracy: 0.8446\n",
            "3980: remaining time: 12899.0 | est. epoch loss: 0.1965\n",
            "3980: est. validation accuracy: 0.8271\n",
            "3990: remaining time: 12866.2 | est. epoch loss: 0.1963\n",
            "3990: est. validation accuracy: 0.8545\n",
            "4000: remaining time: 12839.0 | est. epoch loss: 0.1961\n",
            "4000: est. validation accuracy: 0.8134\n",
            "4010: remaining time: 12816.0 | est. epoch loss: 0.1958\n",
            "4010: est. validation accuracy: 0.7998\n",
            "4020: remaining time: 12781.8 | est. epoch loss: 0.1957\n",
            "4020: est. validation accuracy: 0.8421\n",
            "4030: remaining time: 12747.7 | est. epoch loss: 0.1957\n",
            "4030: est. validation accuracy: 0.8470\n",
            "4040: remaining time: 12718.3 | est. epoch loss: 0.1955\n",
            "4040: est. validation accuracy: 0.7993\n",
            "4050: remaining time: 12688.8 | est. epoch loss: 0.1953\n",
            "4050: est. validation accuracy: 0.8040\n",
            "4060: remaining time: 12660.3 | est. epoch loss: 0.1951\n",
            "4060: est. validation accuracy: 0.7793\n",
            "4070: remaining time: 12644.0 | est. epoch loss: 0.1948\n",
            "4070: est. validation accuracy: 0.7727\n",
            "4080: remaining time: 12618.3 | est. epoch loss: 0.1947\n",
            "4080: est. validation accuracy: 0.7929\n",
            "4090: remaining time: 12579.7 | est. epoch loss: 0.1946\n",
            "4090: est. validation accuracy: 0.7680\n",
            "4100: remaining time: 12559.5 | est. epoch loss: 0.1944\n",
            "4100: est. validation accuracy: 0.8425\n",
            "4110: remaining time: 12535.2 | est. epoch loss: 0.1942\n",
            "4110: est. validation accuracy: 0.8350\n",
            "4120: remaining time: 12512.8 | est. epoch loss: 0.1939\n",
            "4120: est. validation accuracy: 0.7798\n",
            "4130: remaining time: 12483.0 | est. epoch loss: 0.1937\n",
            "4130: est. validation accuracy: 0.8309\n",
            "4140: remaining time: 12468.7 | est. epoch loss: 0.1933\n",
            "4140: est. validation accuracy: 0.8418\n",
            "4150: remaining time: 12451.4 | est. epoch loss: 0.1930\n",
            "4150: est. validation accuracy: 0.8048\n",
            "4160: remaining time: 12428.7 | est. epoch loss: 0.1927\n",
            "4160: est. validation accuracy: 0.7695\n",
            "4170: remaining time: 12396.0 | est. epoch loss: 0.1926\n",
            "4170: est. validation accuracy: 0.8089\n",
            "4180: remaining time: 12369.1 | est. epoch loss: 0.1923\n",
            "4180: est. validation accuracy: 0.7912\n",
            "4190: remaining time: 12339.2 | est. epoch loss: 0.1921\n",
            "4190: est. validation accuracy: 0.7734\n",
            "4200: remaining time: 12310.2 | est. epoch loss: 0.1919\n",
            "4200: est. validation accuracy: 0.7668\n",
            "4210: remaining time: 12281.3 | est. epoch loss: 0.1917\n",
            "4210: est. validation accuracy: 0.7847\n",
            "4220: remaining time: 12254.4 | est. epoch loss: 0.1915\n",
            "4220: est. validation accuracy: 0.7870\n",
            "4230: remaining time: 12230.5 | est. epoch loss: 0.1912\n",
            "4230: est. validation accuracy: 0.8219\n",
            "4240: remaining time: 12207.6 | est. epoch loss: 0.1909\n",
            "4240: est. validation accuracy: 0.8220\n",
            "4250: remaining time: 12177.6 | est. epoch loss: 0.1906\n",
            "4250: est. validation accuracy: 0.7588\n",
            "4260: remaining time: 12143.9 | est. epoch loss: 0.1906\n",
            "4260: est. validation accuracy: 0.8389\n",
            "4270: remaining time: 12120.8 | est. epoch loss: 0.1903\n",
            "4270: est. validation accuracy: 0.8367\n",
            "4280: remaining time: 12091.3 | est. epoch loss: 0.1901\n",
            "4280: est. validation accuracy: 0.8173\n",
            "4290: remaining time: 12058.9 | est. epoch loss: 0.1899\n",
            "4290: est. validation accuracy: 0.8238\n",
            "4300: remaining time: 12027.2 | est. epoch loss: 0.1898\n",
            "4300: est. validation accuracy: 0.8093\n",
            "4310: remaining time: 11998.4 | est. epoch loss: 0.1896\n",
            "4310: est. validation accuracy: 0.8010\n",
            "4320: remaining time: 11977.4 | est. epoch loss: 0.1893\n",
            "4320: est. validation accuracy: 0.8109\n",
            "4330: remaining time: 11953.1 | est. epoch loss: 0.1890\n",
            "4330: est. validation accuracy: 0.8382\n",
            "4340: remaining time: 11935.1 | est. epoch loss: 0.1887\n",
            "4340: est. validation accuracy: 0.8130\n",
            "4350: remaining time: 11909.6 | est. epoch loss: 0.1883\n",
            "4350: est. validation accuracy: 0.8145\n",
            "4360: remaining time: 11886.4 | est. epoch loss: 0.1881\n",
            "4360: est. validation accuracy: 0.8089\n",
            "4370: remaining time: 11864.7 | est. epoch loss: 0.1879\n",
            "4370: est. validation accuracy: 0.7711\n",
            "4380: remaining time: 11838.2 | est. epoch loss: 0.1877\n",
            "4380: est. validation accuracy: 0.8298\n",
            "4390: remaining time: 11812.2 | est. epoch loss: 0.1875\n",
            "4390: est. validation accuracy: 0.8364\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1EmGkDtzJA3v"
      },
      "source": [
        "### Additional Experiments:\n",
        "- Experiment1: No training, directly use Bert pretrained model\n",
        "- Expeirment2: Free transformer layers, only train MLP\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pQne0aq6I4N8"
      },
      "source": [
        "# Experiment1: \n",
        "model = DistilBertForSequenceClassification.from_pretrained(\"distilbert-base-uncased\",num_labels=2)\n",
        "\n",
        "## Use this model to test on dev set  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dk6sbKuKtC-l"
      },
      "source": [
        "# Experiment2: \n",
        "# there are 4 layers in this model\n",
        "import copy\n",
        "model.load_state_dict(torch.load(model_path+\"model1_best.pt\"))\n",
        "model_frozen = copy.deepcopy(model)\n",
        "for param in model_frozen.distilbert.parameters():\n",
        "  param.requires_grad = False\n",
        "\n",
        "## Use the model_frozen to repeat the Train/Evaluate step"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c1S-QCewZztK"
      },
      "source": [
        "## Distance-based approach\n",
        "- Data: query, positive_document, negative_document\n",
        "- Training with triplet loss to compute doc-query distance\n",
        "- Inference: score for each document to the given query \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j3Zw1UoqbNmD"
      },
      "source": [
        "### Training/Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HLY2JvVEnecK"
      },
      "source": [
        "# DistilBertForSequenceClassification\n",
        "# Experiment3: use DistilBertForMaskedLM instead of DistilBertForSequenceClassification in the model variable below\n",
        "# DistilBertForMaskedLM\n",
        "model = DistilBertForSequenceClassification.from_pretrained(\"distilbert-base-uncased\")\n",
        "model.to(device)\n",
        "model.train()\n",
        "\n",
        "optimizer = torch.optim.Adam(model.distilbert.parameters(), lr=LEARNING_RATE)\n",
        "# Experiment1: simply change margin to 0.1 here\n",
        "triplet_loss = nn.TripletMarginLoss(margin=1.0)\n",
        "\n",
        "# Experiment2: using the cosine distance function, use this triple_loss instead of the above one\n",
        "# triplet_loss = nn.TripletMarginWithDistanceLoss(\n",
        "#     distance_function=lambda x, y: 1 - F.cosine_similarity(x, y, dim=-1),\n",
        "#     margin=1.0,\n",
        "# )\n",
        "\n",
        "def predict(inputs, model, tokenizer):\n",
        "    encodings = tokenizer(\n",
        "        inputs,\n",
        "        return_tensors=\"pt\",\n",
        "        truncation=True,\n",
        "        padding=True,\n",
        "        max_length=512,\n",
        "    )\n",
        "    ids, masks = encodings[\"input_ids\"], encodings[\"attention_mask\"]\n",
        "    outputs = model.distilbert(ids.to(device), masks.to(device))\n",
        "    outputs_hidden = outputs.last_hidden_state[:,0] #mean(dim=1)\n",
        "    # Experiment3： using the average of all nodes in the last transfomer layer, use the below outputs_hidden\n",
        "    # outputs_hidden = outputs.last_hidden_state.mean(dim=1)\n",
        "\n",
        "    return outputs_hidden.view(3, len(queries), -1)\n",
        "\n",
        "\n",
        "dataloader = train_dataloader\n",
        "N = len(dataloader)\n",
        "lowest_loss = float(\"inf\")\n",
        "start = time.time()\n",
        "learning_curve_y = []\n",
        "learning_curve_x = []\n",
        "\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    epoch_loss = 0\n",
        "    for i, (queries, pos_docs, neg_docs) in enumerate(dataloader):\n",
        "\n",
        "        optimizer.zero_grad()  # set gradient to zero\n",
        "        anchors, positives, negatives = predict(\n",
        "            inputs=list(queries + pos_docs + neg_docs),\n",
        "            model=model,\n",
        "            tokenizer=tokenizer,\n",
        "        )\n",
        "\n",
        "        loss = triplet_loss(anchors, positives, negatives)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        epoch_loss += float(loss)\n",
        "\n",
        "        if i % 10 == 0:\n",
        "            elapsed_time = time.time() - start\n",
        "            remaining_time = elapsed_time * (1 / (i + 1) * N - 1)\n",
        "            print(\n",
        "                f\"{i}: remaining time: {remaining_time:.1f} | est. epoch loss: {epoch_loss / (i + 1):.4f}\"\n",
        "            )\n",
        "\n",
        "        if i % 10 == 0:\n",
        "            with torch.no_grad():\n",
        "                correct = total = 0\n",
        "                val_start = time.time()\n",
        "                for dq, dp, dn in dev_dataloader:\n",
        "                    anchors, positives, negatives = predict(\n",
        "                        inputs=list(dq + dp + dn),\n",
        "                        model=model,\n",
        "                        tokenizer=tokenizer,\n",
        "                    )\n",
        "\n",
        "                    # using l2 norm distance\n",
        "                    pos_dist = (anchors - positives).norm(dim=-1)  # B distances\n",
        "                    neg_dist = (anchors - negatives).norm(dim=-1)  # B distances\n",
        "                    \n",
        "                    # Experiment2: using cosine distance function: \n",
        "                    # simply comment out the 2 lines above, and use the following 2 lines instead\n",
        "                    # pos_dist = 1 - F.cosine_similarity(anchors, positives, dim=-1)\n",
        "                    # neg_dist = 1 - F.cosine_similarity(anchors, negatives, dim=-1)\n",
        "\n",
        "                    correct += float((pos_dist < neg_dist).sum())\n",
        "                    total += len(dq)\n",
        "                    if time.time() - val_start > 15:\n",
        "                        break\n",
        "                print(f\"{i}: est. validation accuracy: {correct/total:.4f}\")\n",
        "                learning_curve_y.append(correct/total)\n",
        "                learning_curve_x.append(i * batch_size)  # epoch normally \n",
        "\n",
        "        if (epoch_loss / (i + 1)) < lowest_loss:\n",
        "            torch.save(model.state_dict(), \"model2_best.pt\")\n",
        "            lowest_loss = epoch_loss / (i + 1)\n",
        "\n",
        "    print(f\"loss for epoch {epoch} is {epoch_loss}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ne7uZIceNgPD"
      },
      "source": [
        "### Additional Experiments:\n",
        "- Experiment1: Different TripletMarginLoss margins: 1 change: change the `margin` inside `triplet_loss` from 1.0 to 0.1\n",
        "- Experiment2: Different TripletMarginLoss distance functions: cosine instead of l2 norm: 2 changes: \n",
        "  - 1) change `triplet_loss`; \n",
        "  - 2) change `pos_dist` and `neg_dist`\n",
        "- Experiment3: average of final transformer layer instead of just first: 2 changes:\n",
        "  - change `model` to `DistilBertForMaskedLM`\n",
        "  - change `outputs_hidden`\n",
        "- Experiment4: only use pretrained model with no further training (cell below)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kErC8ImL2DtL"
      },
      "source": [
        "# Experiment4: BERT (no training)\n",
        "model2_exp4 = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=2).to(device)\n",
        "# From here, do evaluation on the dev set\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "99qMTrLtarC-"
      },
      "source": [
        "### Testing using distance-based model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "btlxb3qKw9xR",
        "outputId": "d9c4e103-8819-4419-8d42-ee75f91438e6"
      },
      "source": [
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained(\"distilbert-base-uncased\")\n",
        "tokenizer_options = {\n",
        "    \"return_tensors\": \"pt\",\n",
        "    \"truncation\": True,\n",
        "    \"padding\": True,\n",
        "    \"max_length\": 512,\n",
        "}\n",
        "\n",
        "model = DistilBertForMaskedLM.from_pretrained(\"distilbert-base-uncased\")\n",
        "model.load_state_dict(torch.load(model_path + \"demo_model.pt\", map_location=device))\n",
        "model.to(device)\n",
        "model.eval()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DistilBertForMaskedLM(\n",
              "  (distilbert): DistilBertModel(\n",
              "    (embeddings): Embeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (transformer): Transformer(\n",
              "      (layer): ModuleList(\n",
              "        (0): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (1): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (2): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (3): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (4): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (5): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (vocab_transform): Linear(in_features=768, out_features=768, bias=True)\n",
              "  (vocab_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "  (vocab_projector): Linear(in_features=768, out_features=30522, bias=True)\n",
              "  (mlm_loss_fct): CrossEntropyLoss()\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S3nDuT5b5i0G",
        "outputId": "20bb1f58-7a84-48f6-a422-2bc0d0b24962"
      },
      "source": [
        "queries = test_dataset.queries[0:1] \n",
        "documents = test_dataset.positive_doc[0:1] + test_dataset.negative_doc[:4] + [\"UBC MDS is awesome?\"]\n",
        "\n",
        "with torch.no_grad():\n",
        "    inputs = queries + documents\n",
        "\n",
        "    # Tokenize queries and documents \n",
        "    encodings = tokenizer(inputs, **tokenizer_options).to(device)\n",
        "    ids, masks = encodings[\"input_ids\"], encodings[\"attention_mask\"]\n",
        "\n",
        "    # Inference pass through model\n",
        "    outputs = model.distilbert(ids, masks)\n",
        "    outputs_hidden = outputs.last_hidden_state.mean(dim=1) #[:, 0]\n",
        "    vec_queries = outputs_hidden[:len(queries)].unsqueeze(1)\n",
        "    vec_documents = outputs_hidden[len(queries):].unsqueeze(0)\n",
        "\n",
        "    # Compute pairwise distances between queries and documents\n",
        "    all_distances, all_indices = (vec_queries - vec_documents).norm(dim=-1).sort(dim=-1)\n",
        "    \n",
        "    # Print nicely the results\n",
        "    for qdistances, dindices, query in zip(all_distances, all_indices, queries):\n",
        "        # display(Markdown(f\"### {query}\"))\n",
        "        print(query)\n",
        "        for dist, dindex in zip(qdistances, dindices):\n",
        "            # display(Markdown(f\"**{dist:.2f}**: {documents[dindex]}\"))\n",
        "            print(dist.item(), documents[dindex])\n",
        "\n",
        "\n",
        "      \n"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "definition of rent in economics\n",
            "9.83952808380127 Rent, in economics, the income derived from the ownership of land and other free gifts of nature. The neoclassical economist Alfred Marshall, and others after him, chose this definition for technical reasons, even though it is somewhat more restrictive than the meaning given the term in popular usage.\n",
            "11.80168628692627 First, the Rent Court will only award a money judgment for the rent due through the date of trial; a landlord cannot obtain future rent or liquidated damages for rent coming due past the trial date. If the landlord wants to collect future rent, it will have to file a separate collection action.\n",
            "12.555009841918945 The acres of land available for rent (supply) and the number of tenants wanting to rent for cash (demand), as well as production costs and market prices, can affect the amount of rent negotiated. Cash rent paid for Missouri crop and pasture land in 2015.\n",
            "13.303638458251953 Economics is the social science that analyzes the production, distribution, and consumption of goods and services. The term economics comes from the Ancient Greek Î¿á¼°ÎºÎ¿Î½Î¿Î¼Î¯Î± from Î¿á¼¶ÎºÎ¿Ï + Î½ÏÎ¼Î¿Ï, hence rules of the house.\n",
            "15.758798599243164 Although far from unanimous, most mainstream economists would accept some version of Robbins' definition, even though many have raised serious objections to the scope and method of economics, emanating from that definition.\n",
            "17.884113311767578 UBC MDS is awesome?\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}