MRR_scoring.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MRR_scoring.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bff17b3414c44606b94be2509b67360b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f1ad0756bf66470993f4ea7b5a8bfc1e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_953865fe3af346f7acd684bab86b4f6a",
              "IPY_MODEL_e3f8e470b812446f951b0484728c3a92"
            ]
          }
        },
        "f1ad0756bf66470993f4ea7b5a8bfc1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "953865fe3af346f7acd684bab86b4f6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d3188a9331e34878922ed2093a224c08",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f218224e09a4e16b54746dd31c0981e"
          }
        },
        "e3f8e470b812446f951b0484728c3a92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_68f1cbd5adba47d486efd6a1e4ec77fa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:36&lt;00:00, 6.36kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e07ab61861244b7e985085067cab1012"
          }
        },
        "d3188a9331e34878922ed2093a224c08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f218224e09a4e16b54746dd31c0981e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "68f1cbd5adba47d486efd6a1e4ec77fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e07ab61861244b7e985085067cab1012": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7db7d9f6ad14d8f894a39a43b8b2214": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ce1a1c87a21042e69285bedb006cf536",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_07f79a7a152842df8749ea2307320bdf",
              "IPY_MODEL_fffc06ec5c8b4c5d93ce321dd3e60dba"
            ]
          }
        },
        "ce1a1c87a21042e69285bedb006cf536": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "07f79a7a152842df8749ea2307320bdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_93a3e7ebb80c4026ab87bbce7e9e4b7b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58d9f0556c2d4f0ab1923a9a3239a8af"
          }
        },
        "fffc06ec5c8b4c5d93ce321dd3e60dba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1abd05a44b0742258b823ca5d9498c9a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:04&lt;00:00, 103kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_af2ef6df0e7d4a1dbd25b839e0b103c0"
          }
        },
        "93a3e7ebb80c4026ab87bbce7e9e4b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58d9f0556c2d4f0ab1923a9a3239a8af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1abd05a44b0742258b823ca5d9498c9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "af2ef6df0e7d4a1dbd25b839e0b103c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e79f17f3538e495bac908344427beaf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1959086285df4d26947290f215d031a9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c2aace5a2cb44805ba3fd613c20a9110",
              "IPY_MODEL_73d9aec8bdfe4df2bccca3df470bfb6f"
            ]
          }
        },
        "1959086285df4d26947290f215d031a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c2aace5a2cb44805ba3fd613c20a9110": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_23e9620d8ac04f56ae12310d8c275f71",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8d3df215e89c493790a65f51650a1091"
          }
        },
        "73d9aec8bdfe4df2bccca3df470bfb6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ef90177fc2194a56a26099e8edf0cb39",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:01&lt;00:00, 24.2B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2b2a68f386404099958f77ffe88d3e74"
          }
        },
        "23e9620d8ac04f56ae12310d8c275f71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8d3df215e89c493790a65f51650a1091": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef90177fc2194a56a26099e8edf0cb39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2b2a68f386404099958f77ffe88d3e74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c25654fe93ac4c39b09079df7045462c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_759ce1d4b9d441a09b3318aac09a678b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7f9a526cc35544cebd13cb7370607895",
              "IPY_MODEL_2a8cbd358e40455f8f451722f9c2355a"
            ]
          }
        },
        "759ce1d4b9d441a09b3318aac09a678b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7f9a526cc35544cebd13cb7370607895": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f62c3672260242fd83c79afcf920c33a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 442,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e297af2d99b5422c8f993d1f7ac70f6a"
          }
        },
        "2a8cbd358e40455f8f451722f9c2355a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fd986fa33b274625b0be7f8f9754635e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442/442 [00:30&lt;00:00, 14.7B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ecc1cfd1d6e84039a37d501c14cb16e6"
          }
        },
        "f62c3672260242fd83c79afcf920c33a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e297af2d99b5422c8f993d1f7ac70f6a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd986fa33b274625b0be7f8f9754635e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ecc1cfd1d6e84039a37d501c14cb16e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56a488f8ef9943e88fe5945a51157fef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d33652b58c974692b8ab6b8ac5217698",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a3f0faf5fd35402f8b9f8d38792ccf70",
              "IPY_MODEL_e618ab20f7eb4c67ab43b57fe5e350cc"
            ]
          }
        },
        "d33652b58c974692b8ab6b8ac5217698": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a3f0faf5fd35402f8b9f8d38792ccf70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_35ad72de43574c5e8425782a84b3362c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 267967963,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 267967963,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_953faa0441c94d5888f574b241ce0f2f"
          }
        },
        "e618ab20f7eb4c67ab43b57fe5e350cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d9290c3afb4a4729aa7150e2ef5457d7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 268M/268M [00:04&lt;00:00, 60.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_96ca619aeb884da3b45817597c2dff34"
          }
        },
        "35ad72de43574c5e8425782a84b3362c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "953faa0441c94d5888f574b241ce0f2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d9290c3afb4a4729aa7150e2ef5457d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "96ca619aeb884da3b45817597c2dff34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VqcvAhu-PU1I",
        "outputId": "6eb1a6dc-b7aa-48d6-ec6d-1bab2bd20e22"
      },
      "source": [
        "!pip install transformers\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 5.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 51.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 57.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.45 tokenizers-0.10.2 transformers-4.5.1\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7ehBnBPKPV1A",
        "outputId": "b1c76f47-9187-464c-c68f-f43ac47a2b18"
      },
      "source": [
        "import json\n",
        "import sys\n",
        "import numpy as np\n",
        "import time\n",
        "import torch\n",
        "import os\n",
        "import torch.nn as nn\n",
        "import pdb\n",
        "import time\n",
        "import gc\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from sklearn.metrics import accuracy_score\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "print(sys.path)\n",
        "import pandas as pd\n",
        "from collections import defaultdict\n",
        "\n",
        "from transformers import (\n",
        "    DistilBertTokenizer,\n",
        "    DistilBertTokenizerFast,\n",
        "    DistilBertForSequenceClassification,\n",
        "    DistilBertForMaskedLM,\n",
        ")"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['', '/content', '/env/python', '/usr/lib/python37.zip', '/usr/lib/python3.7', '/usr/lib/python3.7/lib-dynload', '/usr/local/lib/python3.7/dist-packages', '/usr/lib/python3/dist-packages', '/usr/local/lib/python3.7/dist-packages/IPython/extensions', '/root/.ipython']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "bff17b3414c44606b94be2509b67360b",
            "f1ad0756bf66470993f4ea7b5a8bfc1e",
            "953865fe3af346f7acd684bab86b4f6a",
            "e3f8e470b812446f951b0484728c3a92",
            "d3188a9331e34878922ed2093a224c08",
            "1f218224e09a4e16b54746dd31c0981e",
            "68f1cbd5adba47d486efd6a1e4ec77fa",
            "e07ab61861244b7e985085067cab1012",
            "c7db7d9f6ad14d8f894a39a43b8b2214",
            "ce1a1c87a21042e69285bedb006cf536",
            "07f79a7a152842df8749ea2307320bdf",
            "fffc06ec5c8b4c5d93ce321dd3e60dba",
            "93a3e7ebb80c4026ab87bbce7e9e4b7b",
            "58d9f0556c2d4f0ab1923a9a3239a8af",
            "1abd05a44b0742258b823ca5d9498c9a",
            "af2ef6df0e7d4a1dbd25b839e0b103c0",
            "e79f17f3538e495bac908344427beaf4",
            "1959086285df4d26947290f215d031a9",
            "c2aace5a2cb44805ba3fd613c20a9110",
            "73d9aec8bdfe4df2bccca3df470bfb6f",
            "23e9620d8ac04f56ae12310d8c275f71",
            "8d3df215e89c493790a65f51650a1091",
            "ef90177fc2194a56a26099e8edf0cb39",
            "2b2a68f386404099958f77ffe88d3e74",
            "c25654fe93ac4c39b09079df7045462c",
            "759ce1d4b9d441a09b3318aac09a678b",
            "7f9a526cc35544cebd13cb7370607895",
            "2a8cbd358e40455f8f451722f9c2355a",
            "f62c3672260242fd83c79afcf920c33a",
            "e297af2d99b5422c8f993d1f7ac70f6a",
            "fd986fa33b274625b0be7f8f9754635e",
            "ecc1cfd1d6e84039a37d501c14cb16e6",
            "56a488f8ef9943e88fe5945a51157fef",
            "d33652b58c974692b8ab6b8ac5217698",
            "a3f0faf5fd35402f8b9f8d38792ccf70",
            "e618ab20f7eb4c67ab43b57fe5e350cc",
            "35ad72de43574c5e8425782a84b3362c",
            "953faa0441c94d5888f574b241ce0f2f",
            "d9290c3afb4a4729aa7150e2ef5457d7",
            "96ca619aeb884da3b45817597c2dff34"
          ]
        },
        "id": "ghTsq_oSN7mG",
        "outputId": "297268db-4d00-4859-d137-bb819876f417"
      },
      "source": [
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained(\"distilbert-base-uncased\")\n",
        "tokenizer_options = {\n",
        "    \"return_tensors\": \"pt\",\n",
        "    \"truncation\": True,\n",
        "    \"padding\": True,\n",
        "    \"max_length\": 512,\n",
        "}\n",
        "\n",
        "model = DistilBertForMaskedLM.from_pretrained(\"distilbert-base-uncased\")\n",
        "model.load_state_dict(torch.load(\"/content/drive/MyDrive/585/demo_model.pt\", map_location=device))\n",
        "model.to(device)\n",
        "model.eval()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bff17b3414c44606b94be2509b67360b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c7db7d9f6ad14d8f894a39a43b8b2214",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e79f17f3538e495bac908344427beaf4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c25654fe93ac4c39b09079df7045462c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "56a488f8ef9943e88fe5945a51157fef",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=267967963.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DistilBertForMaskedLM(\n",
              "  (distilbert): DistilBertModel(\n",
              "    (embeddings): Embeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (transformer): Transformer(\n",
              "      (layer): ModuleList(\n",
              "        (0): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (1): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (2): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (3): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (4): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (5): TransformerBlock(\n",
              "          (attention): MultiHeadSelfAttention(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          )\n",
              "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (ffn): FFN(\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          )\n",
              "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (vocab_transform): Linear(in_features=768, out_features=768, bias=True)\n",
              "  (vocab_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "  (vocab_projector): Linear(in_features=768, out_features=30522, bias=True)\n",
              "  (mlm_loss_fct): CrossEntropyLoss()\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "xTgkRJuw_r31",
        "outputId": "2971fea2-faeb-4e8f-fb80-81c5e84d98df"
      },
      "source": [
        "# preparing for the queries dict\n",
        "\n",
        "diverse_queries = pd.read_csv(\"/content/drive/MyDrive/585/diverse.queries.all.tsv\", sep=\"\\t\", header=None)\n",
        "\n",
        "query_dict = defaultdict(str)\n",
        "passage_dict = defaultdict(str)\n",
        "\n",
        "for qid, q in zip(diverse_queries[0], diverse_queries[1]):\n",
        "    query_dict[qid] = q\n",
        "\n",
        "diverse_queries.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1006418</td>\n",
              "      <td>which cal state schools have football teams</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>291569</td>\n",
              "      <td>how many pairs of chromosomes in a human cell</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>736625</td>\n",
              "      <td>what is cultured stone</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1043711</td>\n",
              "      <td>who make the land rover</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>16151</td>\n",
              "      <td>amara name meaning</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         0                                              1\n",
              "0  1006418    which cal state schools have football teams\n",
              "1   291569  how many pairs of chromosomes in a human cell\n",
              "2   736625                         what is cultured stone\n",
              "3  1043711                        who make the land rover\n",
              "4    16151                             amara name meaning"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "zA0qAV9v__CD",
        "outputId": "f0aff98e-2065-41fc-9566-fba46d2b3123"
      },
      "source": [
        "# preparing for the passages dict\n",
        "\n",
        "diverse_passages = pd.read_csv(\"/content/drive/MyDrive/585/diverse.passages.all.tsv\", sep=\"\\t\", header=None)\n",
        "\n",
        "query_dict = defaultdict(str)\n",
        "passage_dict = defaultdict(str)\n",
        "\n",
        "# preparing passage dict and query dict from top 1000 dev\n",
        "for pid, p in zip(diverse_passages[0], diverse_passages[1]):\n",
        "    passage_dict[pid] = p\n",
        "    \n",
        "diverse_passages.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>140</td>\n",
              "      <td>Charlottesville, VA: CBSA Type: Metro: CBSA Di...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>222</td>\n",
              "      <td>For example, the pressure of electromagnetic r...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>223</td>\n",
              "      <td>The evolution of a gas, a color change, the fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>228</td>\n",
              "      <td>In physics, a photon is usually denoted by the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>264</td>\n",
              "      <td>Lakewood Village Population by Race. The table...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     0                                                  1\n",
              "0  140  Charlottesville, VA: CBSA Type: Metro: CBSA Di...\n",
              "1  222  For example, the pressure of electromagnetic r...\n",
              "2  223  The evolution of a gas, a color change, the fo...\n",
              "3  228  In physics, a photon is usually denoted by the...\n",
              "4  264  Lakewood Village Population by Race. The table..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QlmFHohICZCm"
      },
      "source": [
        "# preparing for the relevant and test set\n",
        "\n",
        "test_triplets = pd.read_csv(\"/content/drive/MyDrive/585/diverse.triplets.test.tsv\", sep=\"\\t\", header=None)\n",
        "test_triplets\n",
        "\n",
        "small_dict_rel = defaultdict(list)\n",
        "\n",
        "for qid, pid in zip(test_triplets[0], test_triplets[1]):\n",
        "    if pid not in list(small_dict_rel[qid]):\n",
        "      small_dict_rel[qid].append(pid)\n",
        "\n",
        "small_dict_test = defaultdict(list)\n",
        "\n",
        "for qid, pos,neg in zip(test_triplets[0], test_triplets[1],test_triplets[2]):\n",
        "    if pos not in list(small_dict_test[qid]):\n",
        "      small_dict_test[qid].append(pos)\n",
        "    if neg not in list(small_dict_test[qid]):\n",
        "      small_dict_test[qid].append(neg)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0uSEIRowVqG"
      },
      "source": [
        "preds_dict = defaultdict(dict)\n",
        "\n",
        "with torch.no_grad():\n",
        "\n",
        "    for idx, (qid, pids) in enumerate(small_dict_test.items()):\n",
        "      docs = [passage_dict[pid] for pid in pids]\n",
        "      queries = [query_dict[qid]]\n",
        "\n",
        "      inputs = list(queries) + list(docs) \n",
        "      encodings = tokenizer(inputs, **tokenizer_options).to(device)\n",
        "      ids, masks = encodings[\"input_ids\"], encodings[\"attention_mask\"]\n",
        "\n",
        "      ids = ids.to(device)  # (Q + D, MAXLEN) \n",
        "      masks = masks.to(device)\n",
        "\n",
        "      outputs = model.distilbert(ids, masks)\n",
        "      outputs_hidden = outputs.last_hidden_state.mean(dim=1) #[:, 0]\n",
        "\n",
        "      vec_queries = outputs_hidden[:len(queries)].unsqueeze(1)\n",
        "      vec_documents = outputs_hidden[len(queries):].unsqueeze(0)\n",
        "\n",
        "      # Compute pairwise distances between queries and documents\n",
        "      all_distances, all_indices = (vec_queries - vec_documents).norm(dim=-1).sort(dim=-1)\n",
        "      \n",
        "      # Print nicely the results\n",
        "      for qdistances, dindices, query in zip(all_distances, all_indices, queries):\n",
        "          rank_dict = preds_dict[qid]\n",
        "          for dist, dindex in zip(qdistances, dindices):\n",
        "              rank_dict[dist.item()] = small_dict_test[qid][dindex]\n",
        "          preds_dict[qid] = rank_dict\n",
        "\n",
        "      del outputs, ids, masks, outputs_hidden, encodings, all_distances, all_indices\n",
        "      "
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y3GH2Wy8qI4O"
      },
      "source": [
        "def compute_metrics(qids_to_relevant_passageids, qids_to_ranked_candidate_passages, MaxMRRRank):\n",
        "    \"\"\"Compute MRR metric\n",
        "    Args:    \n",
        "    p_qids_to_relevant_passageids (dict): dictionary of query-passage mapping\n",
        "        Dict as read in with load_reference or load_reference_from_stream\n",
        "    p_qids_to_ranked_candidate_passages (dict): dictionary of query-passage candidates\n",
        "    Returns:\n",
        "        dict: dictionary of metrics {'MRR': <MRR Score>}\n",
        "    \"\"\"\n",
        "    all_scores = {}\n",
        "    MRR = 0\n",
        "    qids_with_relevant_passages = 0\n",
        "    ranking = []\n",
        "    for qid in qids_to_ranked_candidate_passages:\n",
        "        if qid in qids_to_relevant_passageids:\n",
        "            ranking.append(0)\n",
        "            target_pid = qids_to_relevant_passageids[qid]\n",
        "            candidate_pid = qids_to_ranked_candidate_passages[qid]\n",
        "\n",
        "            if len(candidate_pid) > 0 :\n",
        "              if len(candidate_pid) < MaxMRRRank:\n",
        "                MaxMRRRank = len(candidate_pid)\n",
        "\n",
        "              for i in range(0,MaxMRRRank):\n",
        "                if candidate_pid[i] in target_pid:\n",
        "                    MRR += 1/(i + 1)\n",
        "                    ranking.pop()\n",
        "                    ranking.append(i+1)\n",
        "                    break\n",
        "    print(MRR)\n",
        "    if len(ranking) == 0:\n",
        "        raise IOError(\"No matching QIDs found. Are you sure you are scoring the evaluation set?\")\n",
        "    \n",
        "    MRR = MRR/len(qids_to_relevant_passageids)\n",
        "    all_scores['MRR @20'] = MRR\n",
        "    all_scores['QueriesRanked'] = len(qids_to_ranked_candidate_passages)\n",
        "    return all_scores"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XmcdgTWSNlAg"
      },
      "source": [
        "# preparing the final dict: qid,pid by sorting all ranking for each qid\n",
        "pred_dict_final = defaultdict(list)\n",
        "\n",
        "for key in preds_dict.keys():\n",
        "  rank_dict = preds_dict[key]\n",
        "  \n",
        "  for pred_key in rank_dict.keys():\n",
        "    pred_dict_final[key].append(rank_dict[pred_key])"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0UcMTzsBBIBY",
        "outputId": "fe129090-26fa-4030-c089-7f9bc2c0b325"
      },
      "source": [
        "MaxMRRRank = 20\n",
        "compute_metrics(small_dict_rel, pred_dict_final, MaxMRRRank)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "89.99732924883872\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'MRR @20': 0.1785661294619816, 'QueriesRanked': 504}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g2ntlI9pDrXZ",
        "outputId": "e03fc44a-a75b-4ff8-8c53-e4ac19ba40be"
      },
      "source": [
        "MaxMRRRank = 10\n",
        "compute_metrics(small_dict_rel, pred_dict_final, MaxMRRRank)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "74.3436507936509\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'MRR @10': 0.14750724363819623, 'QueriesRanked': 504}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    }
  ]
}