T5_data_to_text.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "WebNLG -Transformer -T5.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e64389198303458ea65b1514e6ae91ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_541a8f948d994cefa96b434086c000da",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fcb437074f2640e4b6716938853c7cda",
              "IPY_MODEL_dc104e2c6a73446ba0d238ea90952ff1"
            ]
          }
        },
        "541a8f948d994cefa96b434086c000da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fcb437074f2640e4b6716938853c7cda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_efbb05a8150540d5b63806da094242bf",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 791656,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791656,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e9c6aa787d7e443197aa0de89849c2cf"
          }
        },
        "dc104e2c6a73446ba0d238ea90952ff1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e815f78cb5b04361bcbf621a8fa8b0b7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 792k/792k [00:00&lt;00:00, 4.04MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2654217c4c064994bca349a6f20b94e6"
          }
        },
        "efbb05a8150540d5b63806da094242bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e9c6aa787d7e443197aa0de89849c2cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e815f78cb5b04361bcbf621a8fa8b0b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2654217c4c064994bca349a6f20b94e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0c7e75201e6e4c04963b57cabcfc525c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fe888ee5cc15454a88914f010c18d389",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4eab4a545af4430aa0953f1a9897001d",
              "IPY_MODEL_7ca1625e5c904128a6b90424519542ff"
            ]
          }
        },
        "fe888ee5cc15454a88914f010c18d389": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4eab4a545af4430aa0953f1a9897001d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3b146d520e59471eb9092d53ad685523",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1389353,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1389353,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5c02331cdad45b19953a0f985e3d961"
          }
        },
        "7ca1625e5c904128a6b90424519542ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2a5c927f4a5c4d959d26fb806a3c8980",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.39M/1.39M [00:22&lt;00:00, 62.5kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_964af578fa5f4822a9bf7af8c6fb7b02"
          }
        },
        "3b146d520e59471eb9092d53ad685523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5c02331cdad45b19953a0f985e3d961": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a5c927f4a5c4d959d26fb806a3c8980": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "964af578fa5f4822a9bf7af8c6fb7b02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "615577220759413abe6a5cfe30470dc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_29904c8a6734471b9a780137ae862de5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_15f6f633519a4d8d92664176eb1aeb76",
              "IPY_MODEL_b1b232b5750447009d910214e4a4c6b9"
            ]
          }
        },
        "29904c8a6734471b9a780137ae862de5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "15f6f633519a4d8d92664176eb1aeb76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3d49aa452b7248d7ad3b56486c94e58b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1199,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1199,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7efc99954eb44bcf85989d71f2c15eac"
          }
        },
        "b1b232b5750447009d910214e4a4c6b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8d9ecc02356342b99f597204a93a283f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.20k/1.20k [00:00&lt;00:00, 6.30kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_19aef11e25994d86b72777cc2e47c81c"
          }
        },
        "3d49aa452b7248d7ad3b56486c94e58b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7efc99954eb44bcf85989d71f2c15eac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d9ecc02356342b99f597204a93a283f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "19aef11e25994d86b72777cc2e47c81c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f56f513e403b4e89abd171094c93d204": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a51d3f9d1e0c45cf9de5a96ccfe3a826",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2f0903e4d6c7480a86bab857a75f360f",
              "IPY_MODEL_e48d4068acdc445e99b34b74e452d70d"
            ]
          }
        },
        "a51d3f9d1e0c45cf9de5a96ccfe3a826": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f0903e4d6c7480a86bab857a75f360f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9bb04e2e16694ba4bad5e4faba6e6179",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 891691430,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 891691430,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d03a74b33b7943aa8c694ec8a660fafe"
          }
        },
        "e48d4068acdc445e99b34b74e452d70d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_01ad9c215fae4296bc9e72b1b3c4fa64",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 892M/892M [00:21&lt;00:00, 40.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d28ebddb437e4e68bdd3da8cdae1991c"
          }
        },
        "9bb04e2e16694ba4bad5e4faba6e6179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d03a74b33b7943aa8c694ec8a660fafe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "01ad9c215fae4296bc9e72b1b3c4fa64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d28ebddb437e4e68bdd3da8cdae1991c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Viuf5RhzKG00"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-gogiO-IKF-e"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vYq5R3XiKKDf"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MetMTAcsKKG_"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sf1uUqSEKKgk"
      },
      "source": [
        "## Installing the required packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A6ki1-m-5UUA",
        "outputId": "f82515e3-5863-43ea-9864-b17d8d0dd533"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/91/61d69d58a1af1bd81d9ca9d62c90a6de3ab80d77f27c5df65d9a2c1f5626/transformers-4.5.0-py3-none-any.whl (2.1MB)\n",
            "\r\u001b[K     |▏                               | 10kB 20.3MB/s eta 0:00:01\r\u001b[K     |▎                               | 20kB 28.4MB/s eta 0:00:01\r\u001b[K     |▌                               | 30kB 23.7MB/s eta 0:00:01\r\u001b[K     |▋                               | 40kB 27.2MB/s eta 0:00:01\r\u001b[K     |▊                               | 51kB 25.2MB/s eta 0:00:01\r\u001b[K     |█                               | 61kB 27.9MB/s eta 0:00:01\r\u001b[K     |█                               | 71kB 18.6MB/s eta 0:00:01\r\u001b[K     |█▏                              | 81kB 19.6MB/s eta 0:00:01\r\u001b[K     |█▍                              | 92kB 17.3MB/s eta 0:00:01\r\u001b[K     |█▌                              | 102kB 18.3MB/s eta 0:00:01\r\u001b[K     |█▊                              | 112kB 18.3MB/s eta 0:00:01\r\u001b[K     |█▉                              | 122kB 18.3MB/s eta 0:00:01\r\u001b[K     |██                              | 133kB 18.3MB/s eta 0:00:01\r\u001b[K     |██▏                             | 143kB 18.3MB/s eta 0:00:01\r\u001b[K     |██▎                             | 153kB 18.3MB/s eta 0:00:01\r\u001b[K     |██▍                             | 163kB 18.3MB/s eta 0:00:01\r\u001b[K     |██▋                             | 174kB 18.3MB/s eta 0:00:01\r\u001b[K     |██▊                             | 184kB 18.3MB/s eta 0:00:01\r\u001b[K     |███                             | 194kB 18.3MB/s eta 0:00:01\r\u001b[K     |███                             | 204kB 18.3MB/s eta 0:00:01\r\u001b[K     |███▏                            | 215kB 18.3MB/s eta 0:00:01\r\u001b[K     |███▍                            | 225kB 18.3MB/s eta 0:00:01\r\u001b[K     |███▌                            | 235kB 18.3MB/s eta 0:00:01\r\u001b[K     |███▋                            | 245kB 18.3MB/s eta 0:00:01\r\u001b[K     |███▉                            | 256kB 18.3MB/s eta 0:00:01\r\u001b[K     |████                            | 266kB 18.3MB/s eta 0:00:01\r\u001b[K     |████▏                           | 276kB 18.3MB/s eta 0:00:01\r\u001b[K     |████▎                           | 286kB 18.3MB/s eta 0:00:01\r\u001b[K     |████▍                           | 296kB 18.3MB/s eta 0:00:01\r\u001b[K     |████▋                           | 307kB 18.3MB/s eta 0:00:01\r\u001b[K     |████▊                           | 317kB 18.3MB/s eta 0:00:01\r\u001b[K     |████▉                           | 327kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████                           | 337kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 348kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 358kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 368kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 378kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 389kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████                          | 399kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████                          | 409kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 419kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 430kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 440kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 450kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 460kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████                         | 471kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 481kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 491kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 501kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 512kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 522kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████                        | 532kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████                        | 542kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 552kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 563kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 573kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 583kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 593kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████                       | 604kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 614kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 624kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 634kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 645kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 655kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████                      | 665kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████                      | 675kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 686kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 696kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 706kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 716kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 727kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████                     | 737kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 747kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 757kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 768kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 778kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 788kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████                    | 798kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████                    | 808kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 819kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 829kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 839kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 849kB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 860kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 870kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 880kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 890kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 901kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 911kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 921kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 931kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 942kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 952kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 962kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 972kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 983kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 993kB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 1.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 1.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 1.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 1.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 1.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████                | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 1.2MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.3MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 1.4MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.5MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 1.6MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.7MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 1.8MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.9MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 2.0MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 2.1MB 18.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 2.2MB 18.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 54.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 52.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=f4f6fa06fd0d04650dc2d43287bb763579bc54be1cd38b2b9a2f3b48f0303605\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.44 tokenizers-0.10.2 transformers-4.5.0\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 18.9MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.95\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0cxu1nJO4dGT"
      },
      "source": [
        ""
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CalKR6o4KP22"
      },
      "source": [
        "## Importing required libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CB6rH4GenWpN"
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import torch\n",
        "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
        "from transformers.optimization import  Adafactor \n",
        "import time\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4GZK-pXuKUw9"
      },
      "source": [
        ""
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-aWMq3DOKl34"
      },
      "source": [
        "## Loading the processed data "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8CTSGvJQVChn"
      },
      "source": [
        ""
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hq0jrINwVNLr"
      },
      "source": [
        "## Please look at the code [here](https://gist.github.com/MathewAlexander/d4853c2268a2293e479b773f0c317030) to preprocess the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VKZGSyWA0NUk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dcccfda5-037a-4264-e010-93f069bd95d6"
      },
      "source": [
        "cd drive/My\\ Drive/WebNLG2020/"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/WebNLG2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQ2uifxUnVgk"
      },
      "source": [
        "train_df=pd.read_csv('webNLG2020_train.csv', index_col=[0])"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j_dPYbsVK5zO"
      },
      "source": [
        "Trimming off a few data points and so that a batch would not leave any remainder, hence some lines of codes can be avoided (Okay, this might be a hackish way of doing it )"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-WpyPzBKyXf1"
      },
      "source": [
        "train_df=train_df.iloc[  :35000,:]"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WgsIcgFgRCwN"
      },
      "source": [
        "train_df=train_df.sample(frac = 1)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGfaigiqnZW5"
      },
      "source": [
        "batch_size=8\n",
        "num_of_batches=len(train_df)/batch_size\n",
        "num_of_epochs=4"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SQDfdl3mxYf_"
      },
      "source": [
        "num_of_batches=int(num_of_batches)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L_QyyxbkKXf2"
      },
      "source": [
        ""
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9kfjkXbiKeyU"
      },
      "source": [
        "Checking for the GPU availability"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHkcKfKRr1BC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e4e6e27-ac60-47a7-e851-5da78b6e62ed"
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "    dev = torch.device(\"cuda:0\") \n",
        "    print(\"Running on the GPU\")\n",
        "else:\n",
        "    dev = torch.device(\"cpu\")\n",
        "    print(\"Running on the CPU\")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running on the GPU\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_JaUpURdLAla"
      },
      "source": [
        "## Loading the pretrained model and tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xVAXjd6wsOM6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "e64389198303458ea65b1514e6ae91ae",
            "541a8f948d994cefa96b434086c000da",
            "fcb437074f2640e4b6716938853c7cda",
            "dc104e2c6a73446ba0d238ea90952ff1",
            "efbb05a8150540d5b63806da094242bf",
            "e9c6aa787d7e443197aa0de89849c2cf",
            "e815f78cb5b04361bcbf621a8fa8b0b7",
            "2654217c4c064994bca349a6f20b94e6",
            "0c7e75201e6e4c04963b57cabcfc525c",
            "fe888ee5cc15454a88914f010c18d389",
            "4eab4a545af4430aa0953f1a9897001d",
            "7ca1625e5c904128a6b90424519542ff",
            "3b146d520e59471eb9092d53ad685523",
            "d5c02331cdad45b19953a0f985e3d961",
            "2a5c927f4a5c4d959d26fb806a3c8980",
            "964af578fa5f4822a9bf7af8c6fb7b02",
            "615577220759413abe6a5cfe30470dc4",
            "29904c8a6734471b9a780137ae862de5",
            "15f6f633519a4d8d92664176eb1aeb76",
            "b1b232b5750447009d910214e4a4c6b9",
            "3d49aa452b7248d7ad3b56486c94e58b",
            "7efc99954eb44bcf85989d71f2c15eac",
            "8d9ecc02356342b99f597204a93a283f",
            "19aef11e25994d86b72777cc2e47c81c",
            "f56f513e403b4e89abd171094c93d204",
            "a51d3f9d1e0c45cf9de5a96ccfe3a826",
            "2f0903e4d6c7480a86bab857a75f360f",
            "e48d4068acdc445e99b34b74e452d70d",
            "9bb04e2e16694ba4bad5e4faba6e6179",
            "d03a74b33b7943aa8c694ec8a660fafe",
            "01ad9c215fae4296bc9e72b1b3c4fa64",
            "d28ebddb437e4e68bdd3da8cdae1991c"
          ]
        },
        "outputId": "31496177-c0fd-4953-94a8-4e852171cbd2"
      },
      "source": [
        "tokenizer = T5Tokenizer.from_pretrained('t5-base')\n",
        "model = T5ForConditionalGeneration.from_pretrained('t5-base', return_dict=True)\n",
        "#moving the model to device(GPU/CPU)\n",
        "model.to(dev)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e64389198303458ea65b1514e6ae91ae",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=791656.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0c7e75201e6e4c04963b57cabcfc525c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1389353.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "615577220759413abe6a5cfe30470dc4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1199.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f56f513e403b4e89abd171094c93d204",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=891691430.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "T5ForConditionalGeneration(\n",
              "  (shared): Embedding(32128, 768)\n",
              "  (encoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (decoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseReluDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dx0uCmTvLJFb"
      },
      "source": [
        ""
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C4oqM6giLLg4"
      },
      "source": [
        "## Initializing the Adafactor optimizer with parameter values suggested for t5"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1KmOzTQj0E7L"
      },
      "source": [
        "\n",
        "optimizer = Adafactor(\n",
        "    model.parameters(),\n",
        "    lr=1e-3,\n",
        "    eps=(1e-30, 1e-3),\n",
        "    clip_threshold=1.0,\n",
        "    decay_rate=-0.8,\n",
        "    beta1=None,\n",
        "    weight_decay=0.0,\n",
        "    relative_step=False,\n",
        "    scale_parameter=False,\n",
        "    warmup_init=False\n",
        ")\n",
        "\n",
        "  \n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ggfjCkdLcar"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wXw1NKtS1YV3"
      },
      "source": [
        "\n",
        "from IPython.display import HTML, display\n",
        "\n",
        "def progress(loss,value, max=100):\n",
        "    return HTML(\"\"\" Batch loss :{loss}\n",
        "        <progress\n",
        "            value='{value}'\n",
        "            max='{max}',\n",
        "            style='width: 100%'\n",
        "        >\n",
        "            {value}\n",
        "        </progress>\n",
        "    \"\"\".format(loss=loss,value=value, max=max))"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6i7zx4DmC_ap"
      },
      "source": [
        "num_of_epochs=1"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hMe1hKshLgJn"
      },
      "source": [
        "## Training the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qTvda_lWx2nC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "d7124095-4532-4acf-c884-ba3725ed5e00"
      },
      "source": [
        "#Sets the module in training mode\n",
        "model.train()\n",
        "\n",
        "loss_per_10_steps=[]\n",
        "for epoch in range(1,num_of_epochs+1):\n",
        "  print('Running epoch: {}'.format(epoch))\n",
        "  \n",
        "  running_loss=0\n",
        "\n",
        "  out = display(progress(1, num_of_batches+1), display_id=True)\n",
        "  for i in range(num_of_batches):\n",
        "    inputbatch=[]\n",
        "    labelbatch=[]\n",
        "    new_df=train_df[i*batch_size:i*batch_size+batch_size]\n",
        "    for indx,row in new_df.iterrows():\n",
        "      input = 'WebNLG: '+row['input_text']+'</s>' \n",
        "      labels = row['target_text']+'</s>'   \n",
        "      inputbatch.append(input)\n",
        "      labelbatch.append(labels)\n",
        "    inputbatch=tokenizer.batch_encode_plus(inputbatch,padding=True,max_length=400,return_tensors='pt')[\"input_ids\"]\n",
        "    labelbatch=tokenizer.batch_encode_plus(labelbatch,padding=True,max_length=400,return_tensors=\"pt\") [\"input_ids\"]\n",
        "    inputbatch=inputbatch.to(dev)\n",
        "    labelbatch=labelbatch.to(dev)\n",
        "\n",
        "    # clear out the gradients of all Variables \n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # Forward propogation\n",
        "    outputs = model(input_ids=inputbatch, labels=labelbatch)\n",
        "    loss = outputs.loss\n",
        "    loss_num=loss.item()\n",
        "    logits = outputs.logits\n",
        "    running_loss+=loss_num\n",
        "    if i%10 ==0:      \n",
        "      loss_per_10_steps.append(loss_num)\n",
        "    out.update(progress(loss_num,i, num_of_batches+1))\n",
        "\n",
        "    # calculating the gradients\n",
        "    loss.backward()\n",
        "\n",
        "    #updating the params\n",
        "    optimizer.step()\n",
        "    \n",
        "  running_loss=running_loss/int(num_of_batches)\n",
        "  print('Epoch: {} , Running loss: {}'.format(epoch,running_loss))\n",
        "  "
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running epoch: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              " Batch loss :0.2204882651567459\n",
              "        <progress\n",
              "            value='4374'\n",
              "            max='4376',\n",
              "            style='width: 100%'\n",
              "        >\n",
              "            4374\n",
              "        </progress>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 , Running loss: 0.5023154723678316\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UwVEDrdZ545G"
      },
      "source": [
        "## Plotting the loss over time"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7quhDpSxxTGL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "178d30fc-bab5-4b72-dc98-001de870db03"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "   \n",
        "steps = [i*100 for i in range(len(loss_per_10_steps))]\n",
        "  \n",
        "plt.plot(steps, loss_per_10_steps)\n",
        "plt.title('Loss')\n",
        "plt.xlabel('Steps')\n",
        "plt.ylabel('Loss')\n",
        "plt.show()"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEWCAYAAABsY4yMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgU1b3/8fd3VhiGfYZ930VRgZGouCBugEaNS6ImxqiJ1+hN4k9vFDVqTDTR6DVqrsYYjcZEjSvu4oqggOCwySb7vs4wzMrsc35/dHVPbwPD0sxQfF7PwzPdVd1V1UX3p06dc+qUOecQERH/SWrqDRARkcRQwIuI+JQCXkTEpxTwIiI+pYAXEfEpBbyIiE8p4EVEfEoBL4clM1trZmc09XaIJJICXkTEpxTwIh4zSzezR8xss/fvETNL9+Zlmdm7ZlZoZgVm9oWZJXnzbjWzTWZWYmbLzOz0pv0kIgEpTb0BIs3IHcDxwLGAA94CfgPcCdwMbASyvdceDzgzGwz8N3Ccc26zmfUBkg/uZovEpxK8SL0fAr9zzm13zuUB9wBXePOqga5Ab+dctXPuCxcYyKkWSAeGmlmqc26tc25Vk2y9SBQFvEi9bsC6sOfrvGkADwIrgY/MbLWZTQRwzq0EbgR+C2w3s/+YWTdEmgEFvEi9zUDvsOe9vGk450qcczc75/oB5wE3BevanXMvOudO8t7rgAcO7maLxKeAl8NZqpm1CP4DXgJ+Y2bZZpYF3AX8G8DMzjWzAWZmQBGBqpk6MxtsZmO9xtgKoByoa5qPIxJJAS+Hs/cJBHLwXwsgF/gGWAjMBe71XjsQ+AQoBWYCTzjnphCof78fyAe2Ap2A2w7eRxBpmOmGHyIi/qQSvIiITyngRUR8SgEvIuJTCngREZ9qVkMVZGVluT59+jT1ZoiIHDLmzJmT75zLjjevWQV8nz59yM3NberNEBE5ZJjZuobmqYpGRMSnFPAiIj6lgBcR8SkFvIiITyngRUR8SgEvIuJTCngREZ/yRcA/9ukKpi7Pa+rNEBFpVnwR8E98vpLpK/ObejNERJoVXwS8YWhcexGRSP4IeAPlu4hIJH8EPIE7HYuISD1/BLyZSvAiIlH8EfCAUxleRCSCLwIe1cGLiMTwRcBbU2+AiEgz5I+AN3WTFBGJ5ouATzKoU76LiETwRcCbmRpZRUSiJCzgzWywmc0P+1dsZjcmZF2okVVEJFrCbrrtnFsGHAtgZsnAJmBSItZlpgudRESiHawqmtOBVc65Bu/+vX90oZOISLSDFfCXAi/Fm2Fm15pZrpnl5uXt25C/ZqAyvIhIpIQHvJmlAecBr8ab75x7yjmX45zLyc7O3rd1oDp4EZFoB6MEPx6Y65zblqgVaDRJEZFYByPgL6OB6pkDxVA3SRGRaAkNeDNrBZwJvJHY9agELyISLWHdJAGcc2VAx0SuAzQevIhIPP65klUJLyISwRcBDxoPXkQkmi8C3lRHIyISwzcBr3wXEYnkj4BH48GLiETzR8CrBC8iEsMfAY/6wYuIRPNHwJupBC8iEsUnAY/q4EVEovgj4FEVjYhINH8EvO7JKiISwx8Bj0rwIiLR/BHwGk1SRCSGPwJe48GLiMTwR8CrBC8iEsMXAQ+6klVEJJovAl7jwYuIxPJHwAMqw4uIREr0PVnbmdlrZvatmS01sxMSsx7VwYuIREvoPVmBR4HJzrmLzSwNyEjESjSapIhIrIQFvJm1BU4BfgLgnKsCqhKyLo0HLyISI5FVNH2BPOBZM5tnZk+bWavoF5nZtWaWa2a5eXl5+7QileBFRGIlMuBTgBHAX51zw4EyYGL0i5xzTznncpxzOdnZ2fu0Ig1VICISK5EBvxHY6Jyb5T1/jUDgH3gaD15EJEbCAt45txXYYGaDvUmnA0sSsa5ACV4RLyISLtG9aH4BvOD1oFkNXJWIlZglYqkiIoe2hAa8c24+kJPIdYDq4EVE4vHFlaxJuuGHiEgMXwS8GdTVNfVWiIg0L/4IeI0HLyISwxcBj8aiERGJ4YuAN3Qlq4hINH8EvBJeRCSGPwJedfAiIjH8EfCqgxcRieGfgG/qjRARaWb8EfAaD15EJIY/Al4leBGRGL4IeFAdvIhINF8EvGk8eBGRGP4IeFARXkQkij8CXnXwIiIx/BHwqAAvIhLNHwGv8eBFRGL4I+BRCV5EJFpCb9lnZmuBEqAWqHHOJeT2fRqqQEQkVqJvug1wmnMuP5ErUDdJEZFYPqqiUcSLiIRLdMA74CMzm2Nm18Z7gZlda2a5Zpabl5e3TytRFY2ISKxEB/xJzrkRwHjgBjM7JfoFzrmnnHM5zrmc7OzsfVqJxoMXEYmV0IB3zm3y/m4HJgGjErEeleBFRGIlLODNrJWZtQ4+Bs4CFiVmXbqSVUQkWiJ70XQGJplZcD0vOucmJ2JFGg9eRCRWwgLeObcaOCZRy4+gEryISAzfdJNUwouIRPJHwOtCJxGRGP4IeHShk4hINH8EvOrgRURi+CPgUT94EZFo/gh4jQcvIhLDHwGPSvAiItF8EfBoqAIRkRi+CHgL9IQXEZEw/gh4UzdJEZFo/gh41E1SRCSaPwJedfAiIjF8EfBJZtQp4UVEIvgi4HUlq4hILF8EPJiqaEREovgi4E3jBYuIxPBHwKNGVhGRaI0KeO/+qkne40Fmdp6ZpTbyvclmNs/M3t2fDd39OlR+FxGJ1tgS/DSghZl1Bz4CrgCea+R7fwUs3ftNazzdk1VEJFZjA96cc7uAC4EnnHOXAEfu8U1mPYBzgKf3fRMbsXEqwYuIxGh0wJvZCcAPgfe8acmNeN8jwC1A3T5sW6OpDl5EJFZjA/5G4DZgknNusZn1A6bs7g1mdi6w3Tk3Zw+vu9bMcs0sNy8vr5GbE7MMVdGIiERJacyLnHNTgakAXmNrvnPul3t422jgPDObALQA2pjZv51zP4pa9lPAUwA5OTn7nNKKdxGRSI3tRfOimbUxs1bAImCJmf16d+9xzt3mnOvhnOsDXAp8Fh3uB4pptDERkRiNraIZ6pwrBi4APgD6EuhJ0ywYpnwXEYnS2IBP9fq9XwC87ZyrZi/KzM65z51z5+7LBjaGxoMXEYnV2ID/G7AWaAVMM7PeQHGiNmpvqYZGRCRWYxtZHwMeC5u0zsxOS8wm7T2NBy8iEquxjaxtzezhYHdGM/tfAqX5ZsHMcCrDi4hEaGwVzT+AEuD73r9i4NlEbdTe0oVOIiKxGlVFA/R3zl0U9vweM5ufiA3aJxqqQEQkRmNL8OVmdlLwiZmNBsoTs0l7L0lXsoqIxGhsCf464Hkza+s93wlcmZhN2nuqohERidXYXjQLgGPMrI33vNjMbgS+SeTGNZZGkxQRibVXd3RyzhV7V7QC3JSA7dknGg9eRCTW/tyyzw7YVuwnleBFRGLtT8A3m0xVHbyISKzd1sGbWQnxg9yAlgnZon1hzeZkQkSk2dhtwDvnWh+sDdkfwXh3zmEKexERYP+qaJqNYKarmkZEpJ4/At4rwyvfRUTq+SPgQyV4RbyISJA/At77q3gXEannj4BXHbyISIyEBbyZtTCz2Wa2wMwWm9k9CVwXgMaEFxEJ09jBxvZFJTDWOVfq3c/1SzP7wDn3VaJWqBK8iEi9hAW8C7R4lnpPU71/CYlgdX0XEYmV0Dp4M0v2bgyyHfjYOTcrIesJdpNUCV5EJCShAe+cq3XOHQv0AEaZ2VHRrzGza4P3es3Ly9un9YQaWVUHLyISclB60TjnCoEpwLg4855yzuU453Kys7P3afn1QxXs+zaKiPhNInvRZJtZO+9xS+BM4NtErCvJK8LXKeFFREIS2YumK/BPM0smcCB5xTn3biJWVF9FIyIiQYnsRfMNMDxRy4+/zoO5NhGR5s0nV7KqCC8iEs0fAe/9VS8aEZF6/gh4jUUjIhLDHwHv/VW+i4jU80fABwcbUxFeRCTEJwEf+Kt4FxGp54+A9/6qAC8iUs8XAY/GgxcRieGLgA+NFqx8FxEJ8UfAqw5eRCSGPwJe48GLiMTwR8BrPHgRkRj+CHjvr0rwIiL1/BHwqoMXEYnhj4BHV7KKiETzRcCjwcZERGL4IuCDt+wTEZF6vgj4YLzrnqwiIvUSedPtnmY2xcyWmNliM/tV4tYV+Kt8FxGpl8ibbtcANzvn5ppZa2COmX3snFtyoFekXjQiIrESVoJ3zm1xzs31HpcAS4HuiViXetGIiMQ6KHXwZtYHGA7MijPvWjPLNbPcvLy8fVx+4K/iXUSkXsID3swygdeBG51zxdHznXNPOedynHM52dnZ+7UuFeBFROolNODNLJVAuL/gnHsjgevxHinhRUSCEtmLxoBngKXOuYcTtR7QWDQiIvEksgQ/GrgCGGtm871/ExKxItXBi4jESlg3Sefcl4TdbCmRNB68iEgsf1zJqvHgRURi+CPgvb8qwYuI1PNHwGuoAhGRGL4I+GAZXlU0IiL1fBHwKsGLiMTyR8A39QaIiDRD/gh4UzdJEZFo/gh476/q4EVE6vkj4FUHLyISwxcBH7wnq27ZJyJSzxcBj8aiERGJ4YuAD9bBF5VXU1Fd26TbIiLSXPgj4L0qmque/Zpz//JlE2+NiEjz4I+AD3u8cntpk22HiEhz4o+A15VOIiIxfBHwSUp4EZEYvgj4FqnJTb0JIiLNji8CPjM9YTemEhE5ZCXyptv/MLPtZrYoUesIapUeWYJ/e8FmPlmyLdGrFRFp1hJZgn8OGJfA5Ye0Tk+NeP7Ll+bx0+dzD8aqDylfrd7B95+cSXVtXVNviogcBAkLeOfcNKAgUcsPF12CDyqrrDkYqz9k3PzKAmavLWBLYUVTb4qIHARNXgdvZteaWa6Z5ebl5e3TMlKS43+MhZuK9mfTfMcdhLF6SiqqeXHW+oOyLhHZvSYPeOfcU865HOdcTnZ29gFd9qK9CPjSyhrW5pcd0PU3V5U1iRvO4bdvL+H2SQuZteagnLyJyG40ecAnQp+OGbTLSGVNWGB/unQbK7eXhJ5f8Ph0Tv7TZ6HnP3x6FmMe+vxgbuZBFyxTl8cZr6essmavDogN2VFWGVqeyL5IZAHkcOPLgO/cpgV9s1qxOq+MnWVV3D5pIdf8M5czHp4Wes38DYVsKCgPPV+woRCAqpq9b4C84YW5/HPG2v3e7nhW5ZXSZ+J7LN9WsucX70Gw1qS8KvYH9NLs9XzviemU7mcwJ3sXndXWqYpG9t7SLcUM/s1kPly8tak3xRcS2U3yJWAmMNjMNprZNYlaV7TrxvSnb1Yr1uSX8einK3hx1vrQvAufmM7v310Seh5dV3zzqwvYXFhOY5VV1vDewi3c/fZithUf+MbLdxdsAeDt+Zv3e1nB8fLjleDzS6uornWs27F/1VTJSYfH2Pz5pZXM9woFcuDMXb8TgCnfbm/iLfGHRPaiucw519U5l+qc6+GceyZR6wq3+J6zOW1wJ/pnZ7K1uIKSisgS6dz1hTzz5ZrQ83lRP9J3FmzmxPs/49bXvmnU+lbl1Q9u9lnYl3JHaSWPfrJiv7sk1tYF3h8Mzv0RjNx4QyoHq1Q2FOzar3UEt7O69tAN+Fmrd/Dvr9bt9jUXPjGDCx6fvt/r+nptgbqthqn2zqBTkhv3fc8vreSByd9qHzbAd1U0Lb1hC/pntwJg7R5KpBc+MYOFG2Prnl/O3RB6vLvqhhXb6gM+d+1O1u/YhXOOP01exp8/Wc6nSxt/wVVFdS2/fGke5z8+nZtfWQBAjbfulAMR8C64ntgfQzDg1+2oD/h/f7WO6/41Z6/WkeRtZ7xqoL31r5lr+e5BGP55xsp85qyrbxT+wVNf8Zs3d3993nrvQLg/9x9YvLmIS56cyUMfLtvnZRzKPl+2nS9WRPacCxYMUpIaF00PfbiMv36+io8W68LGeHwX8MGAGdylDQAr4tRdd2qdHvF8WQP12zW1dUxbnkf/299vsA58+fYSUpON0wZn8+43mznlwSnc9dZiNhcFqnn2pjfJx0u28faCzSzYUMjrczcC9QeXpAYCfsaqfB6fsrLR64D4VTTBuvd1YSX437y5iMl7WRcarIMvq2pcXf6MVfncPmlh3G6Vd761mIWbihJ+E5fLn57FRX+duU/vLSqv3uf17iwLvPdQ785bVF5Nfmll3Hk7y6oanPeTZ7/mimdmR0yr9s5Y01IaF02pXhfpLUXxq1Vfzd3AE5/v3e9jXzXHXni+CfiJ44cwoFNm6HmvDhm0TE2muCI2aLaXRH7h5nn1fkG3TxgCwJaiCp6cugoIlM4BbnltAfeG1eF/vaaAoV3bMLxXeyq908t/fbUu9PqZq3Y0uM1r8ssY/+gX5Hnb83HU8ApVNXWhcNvlBeaKbSVMW57HXW8torq2jsv/PosHP1zWyEZNrw6+qpbiishgCgby+h2xVTS7C9jgel/J3cC4R6ZR4/1Ad4WV4P/6+Sr6THwv7jb+1/NzeHHWerYUNdx+sbWBea/kbmDOup1x5+1J4a6qiF5V8TTmwFK4a98D3nn/HzV1rsmuGyivqt3vdR//h0/JufcTINCm9cmSbZRX1VJdW8fw338cmteQOet2hr77pXF+r7vTLiNwFXteAweRX7/2DX+anPgzpC9X5DPmoc95a/6mhK9rb/hmlK7rTu3Pdaf2Dz1PTjIGdc5kQZzql2gzV0eG8FHd2wKBAF62NRACwYbXV3IDJeveWa1Yl1/GvA2F3Hj6IHp2aBmxjPLqWlKTjc2F5TjnuOTJmYzq24Fbxg0JvWbOup0s3VLMNxsLGTukEzNW5UcsY1NheaiEGPzin/nn+p5AZw3tEnq8rbiCbu0ityFa8Hf84uz1/O7dJXxy0yms3F7K1OX5lFYGwmx9nDr4wl3VXPzkVPpmteKUgdmUVFRz1ei+fLxkG7e8/g0v/PQ73OK1WezcVQUEDkhF5dUs3lTEA5O/DU1r3SJyWIns1umUVNawYENhg9u/paiCPlmtYqYH17n2/nN2+7kj90FgJ3zviRmsyS9jxX3jG3xtaWXNHkcqDX7eKcu286uX5vHlxLG0ifqMDS7f+z/dUlTOkDsn8/D3j+Wco7tGvOa1ORvJSEtmwrCu8RaxX/JKKjnuvk+469yhXH1S331eTvgZ4au5G7nl9cD/y5AurRv1/kc/XcGiTUWcccQZoe/7rkaeAQYLEvEKJuGcc6E7vyXC0i3FACzYUMT5x3ZP2Hr2lm9K8PEM8appGjJx/BDOGtqZ1XmRp1Z9OgbC5PNleewoC/yAV2wv4ZWwevk731zE01+uwTk4/YhO9GifEbP8UX07UFxRw6w1BeSu28mkeZtwzlFVU8fXawtCvW42FZbz8tcbyC+t4vcXHMXzV48CAmFb6H3h1+7YFdOFMRicwWXE8/XaAt6av4nq2rpQfX7wrlcLNhRx3b/n8tLs9ZR469lUWB7TYFVQVsXGneV8sSKf+95fymOfreSGF+eGfsjhPXy2FQf7wddy/wffcvnTs0LzyiprqYladuc2LQBCPVIqqmv5cPFW6sJK+/FOv6OX01j/8+o3fOcPn4aukQg/O4he5vSV+TzyyfLdLu/Sp77iixV53P7GQooravYYNOGCHQA2FJRTWVPHm3FKf//z6gKuf2HubpezubB8rxvHt5dUhLr2vjFv4169N1x46d85x9SwOvVvtzaua+/GnbsoKKtiQ0F56Pu+qzLy7OmjxVt59JMVLNtawt+nrQ59P0q8M9G563dy43/mUegdcKO3LbwqraqmjtveWBhTpTJz1Q7++P7SiGmT5m3k/g++PWSvzPZNCT6ewXsoQRzdoy3lVbV8FFU10rlNC5KTjLcXBH5w/bJb8eHibXwYpyGnc5t0juzWhq1xukiO6tOR6St38NfPA9U8W4oqWL6tlKe/WM2rczYydkgnAD5avI0vVwZK78N7tiPbayNYt6MsVAUwdXkep//v5xHLD6+73bhzF8f16UB+aSUdMtJISjKKyqv54d9nUVVbx+q8spiGz/Azl3UFu0hNNqprHUu3FNOhVVpo3s2vLoj5bMvCfrwbCwPhkmQQzOVdVTUxZ0YX/XUGO3dVMfO202mVlkx1rQtVDf1t2mq6tm2BmXH324u5bXz9mc6WogoqqmtJTU4K9dIpDPvBVtfWhepioxXtqub7f5vJsB5tOW1wp1DbRlD4LR537qqmfUZ96ftX/5kPBM4O05KTeODDb7ng2O4c0TWy4BBejxw8CDvn+Ozb7bRMTebEAVlxty26mqymto7ctQX0yWpFVmZ6oy74cc5xzT8DA+vdNn4Ia/LLuPLEPkx8/RsGdMrkpyf3i3m9mXHhEzPYuDNw4DT2vmRbUFbF9JX5jOrbITStrKo2VJKNdtWzs+nStgVnH9mFMYM7RVTXBc+O523YSXGoBF/L9pIKvl6zk82F5dznBe/aHWVMmreJTYXlHN2jbeggua24kjfnb+aMoZ059+huABSX1xeIthVX0i4j8J2etWYHL81ez5ItxTxzZQ5ZmYHf2+VPf4VzcMPYAaGzsDvfXExpZQ1DurTmguENl8wdzfMA4O8SfNfYgD+uT/vQ4zYtUukb59Q/Ocno1Dqd/NIqsjLTuHxUrwbXMXZIJ8yMTq1bxMwb2i0QBFOX53F8v8APYdaaHbzmhcySzcHTuvqumoO7tCY7M53u7Vpy11uLI/paB0vH8WwoKOfbrcXk3PsJz85YS22d4+kvVlNVW0dWZhpPTl1FVVQJ9bU59WFXW+cY0Cmwv877v+mc9MCU0Lx4P9rKsAvCvt0SCPvhver37fJtpREXkkHg7GBXVS0bCnbxP68u4Ii7JrOjtIoe7QNVM799Z0noApfHPl0R8b7rX5jLsN9+GCoh7yyrL6nF67v/5Yp8Tvjjp7w6ZwPLtpUwad4mbngxUBIO/qAhsoH9L5+t4ME4PVrySyuZsWoHf5u6mrvfWrzbs4dtxRW8MGsduet2cs0/c7n86VlsL6nAOcc979T/f369tiCmbnjJlmIufnImVz/3NRDZZbUuTvtFaWXg7HDplmKWbinmx/+Yzd1vLya/tJL/fL2Be9+LLI1OXZ5H39veZ01+WSjcIXDLy4KyKv74/lKmLNvOFc/MYltxxW6vRn5uxlp+8dK80LYC5JdUNngGM2VZHi/N3sBPnv2agrIqtpfUF4iCvbrmbygMlbTX7ijjpAemcMOLc0PhDvUl8edmrOWmVxawJOq7GX7ADi90PfH5Sv74wVImL9rKJu+zL9hQGNE+0Nq7r8TysMJLpzaB70p4W1pReTV9Jr7HpLAzn+CBJnj9x6Y4Z1U7SisZff9nB+SK8cbydQk+WEXzX6f0Y+L4IXy4eBvZrdNCPSZat0iJCfjWLQK7pHObFmwpquCIrm24anRfthRVhPrPp6ck8bcrRrKrqpYRXqjF66ce3lvnF2MHsipvPl+v3RmqCw9+AUu8H9KnN58aKoleeWJv/vD+tzTWwx8v5+GPA9UJnyzZxqR5G1m0qZhTBmUzdnA2v31nyR6WAEd0bd1gCSxaaWUNl43qyTsLtrCjrIq0lCR6d8gINXoGg6xXh4yYev0XZq3nTa9aZ1NhOT85sQ9nH9mFy/7+FTO8H1JZ2NnG599uZ7NXlfLvWeu4fcIRFIQF/I+fmc3l3+nFf48dCMDrczaGzjpenB24yO3iET14OXcDnVqn0zerVahnx/0f1O/j52fG7/ueX1rFa3MC1XOz1xZw+6SFDe6Xm19ZQE2dY0SvdqFpHy/ZxpjBnXh2+lqenb6W351/JHe9tTjifW1bpoYO4N9sLOL1ORtZHtYInF9WGVGIqKtznPnwVLYUVUScOQFMmhu/oS/4WaPbegz488fL+ddX6/jbtNUAnPPYF+SXVvHUFSM568hAW8897yzm1EHZjBncKdQxYPHm+u/Lt1tLQtWAu3P+41+GqkGDkpOM+RsKQ2esDVXvLN4cGY7rog4oK8ICPrzq6a2wasSrRveJeM8Vz8zijxcOo2vblhRXlPDE56uorVvJM1fmsN37PwmepU5dHqiOA3jow+V0btOCF75az3sLAxckBs9A/t9/5oPB+KO6sCqvlHsvGMbU5XlsKiznb9NWc/HIHvzqP/OYdstpjW6z2Re+DvgOrdK489yhnDwwCzNj3FFdIurdOmamh07bAKZPHEuG16jWxasb7p+dSXKScee5Q0MBP/XXp9GlbWyJ/b1fnkTr9FRKKqtxDtqHVXPk9GnPMT3a8c6Chq9IDT8gXHNSP0b2bs+24srd1sE+ePHRvDBrfURJf+bqHSQZPPz9Y7jg2O58tCS2q2NWZhr5pVUR04KfNV5vl94dM2J+TFmZ6bTLSKW0sobObdJpmxH7RR3WvW1MwL80e33E8w6t0jihf0cGdspkxfZSzOobhH98Qu+I4J26LI+J44aEGjcBNhdV8NBHy5m2PJ9//XQUz81YS6u0ZMqqalmdV0bvjhl8p18HXs7dQGpyEllR3WT3JK+kkqVb6gMn2NAeTzDg5q4vpHu7lqQmG799ezHVtfX96qPDHQJngpPm1QdzdLXY1qKKUMAv2lTEuWHXB4wekMWcdTtDDY5vh33HLvrrDMYO6cSPju8dOng/O31txLIXbCyK6YwQ/G5MXrSV04Z0YsaqHaED1Oo/TIjb9XHq8sZdfbqhoJwdUd+9E/t3DHQp3sPxId5ZbOc26aHpq7yAX5Nfxt+mro67jOjP/8WKfK55LpdMr3AXvGBx8ebiUJVb8Iznyn/UV8dtKSrnqme/jjibLSyvprKmlvkbC2mfkcoLs9azubCci0b04Cnv4PnOgs2hHFixrZSRvevPfA80X1fRAFxzUl8Gda6vqundMYM7JhzBG9efSGZ6Cm1bptK9XUtuPnMQ3du1DIVy8DqLftn1JY3/OiVQn5mVWR/c4Y7s1pZeHTM4sltbjureNuJ16SnJnNC/IwCne3XvUH8BU0qSRdx6MDnJGNm7Q0TvieFhpcKgi0b0oGeH+gbejt72jx/WlQtH9CApyejUJvZgdPNZg/lBTk+e/NHI0LQ2LVJirhEImvyrU2KmZWWmh+rqO7duQduWkQHfIjUpYv81JFjvfdd3h9K2ZSrXjK7v0XHzmYNDj3+Q05Nl20o4/pe5qtwAABHsSURBVI+f8sgnK2KWM3ttAYN/M5mFm4r46cn9QqfcR3ZrE6ouS0tJiikxPX/1KHp1yOCYnrH7FwINkvF6F3Vv15LfnHNEg59rRO/2nDoou1FX9V57Sj8uHtmjwfnBbqQLNxbxxw8iq17u/u6RoWouiGybmbNuJw9+uIybX5kfmrZyeym9O8Z2Cohn2op8bntjYUSwzV2/M1SCD/fS7A0x0xqyK6o96JxhXamqqYupRmyM8N/36rwyamrrmLosENL/d/lwfnbynnsIrdheEmpw7+xVywTbxbq1bcHmwvKYz1znAlWVpwyqHwW3cFcVS7eUUFVTx7biSlZuL2VXVS3fe2JG3LOSbcUVTF60ZZ87DeyJ7wM+mpnxs1P6hapWIFBy/8XpAyNeV+a14oefFk8cP4QV941vcPz5aOkpyQzp0ppbva6RV57Qm+kTx/L0lTmh1wSP3u1bpTXYjev8YwONRm/8/ETeumF0xLykJAv9uCcM6xIKslMH1n/pusQJ+AlHdeWBi49m3FH1XS1bpadEHGQuPa4n/bNbcfXovrRMS+aiEYEACtZGdcxMC50BdWvXMibgu7VrGXGGFC1YrRX8sZ88MJsFd5/F9acNCL2mbUYqn958KneeO5RfjxvMz8f0Jz01KfRjucQLxZd+dnzEsr/Tr0PoLGto1zYM7NSaS4/ryV8uG05q1GXwowdkMe2W0/jD946Ku513TFpEeXUtE8cP4c8/OIaszHRSk41/Xj0qphET4LJRvTj/2G7cOm5wgw2s4Y2TN505iCFdWvPQJccw6foTMYN7LziKQZ0zecb7rvzixXl89y9f8t3/+5LpK+vrgx+65BgGdMqklff/dnSPthHrOXlgFj8+oTefLI0sXV88Iv7BpGvYmekZR3Qiv7Qyoq0GAm0yeSWV9POqN1ulRXYlDf9s0dsCgQN/uNYtUjg5LCSD1abBwsaxUQfe6O9z/+z661+qaut4YdZ6fvvOEnp3zODco7txxzlDI15/VPc2DAy7ZgYCYb2psJwf5PTks5vHADBteaBH0Mg+HaiudRx3X/z+/OcMq/8NzV1fyAdedU1jXP/CXK7791yeS9BghYddwDfWpcf1BCK/XGbWYG+Nhky+8RR+PibQPz8lOYnu7VpGBPlpXmm+w26C8KFLjmHB3WdhZgzsHPhi/vL0gUz99RggUMIASEtOCpUCw0sV2WGl8jsmHMFNZw6KqE65ZdxgTh6YRU7vDqHQ/f0FR3Hf94bx6c1juOu7gR/IgxcfzZe3nsbRPQL7JDM9hW1eqeekAVmhi06CKqvrYn7M4W6fcARZmWmhOt6g9hmpmNXXlfbPzuSak/qSlZnOreOGMO3Xp4Ve+4cLh/H1HWcwLCzY/uuUfhzft2Mo4I/s1pbkJOP+i47mqO5tSYo6kAY/czBY4jW8AwzqnMn3hvdg+sTTWPjbs0MX1k399Ri+uKV+m645qQ+PXjqcHu0zOHlgFif068iFI+p7YPz67MH865pRoee/PH1g6DsxvFd75t95Fj86vjcf/b9TOf2Izozo1Y6q2rq4V7wGwy54YB4bdnZ45Qm9efTS4Xz3mG4x7wsWBIIeuuQY1t5/DjNvOz007eKRPePuh2+3lpBfWsmJAzrSKi2ZBy85hiPDlvf81aMi/o+GdW/LmzeM5k8XH82w7m35WdRBsXu7lnRr2yL0PR3q9VJ64KKjWXnf+Jizyl4dIs8+iiuqmTh+CHd739O73w5Ugd0+If7ZVXZmOvecf2REYSaoTcsUWqUHzmRnrSnADM4a2jk0P96+POOIzhHPg+0Y0PgxpB6fsvKADO8Rzdd18Ptj/LCue3UBzb46ZWA293/wbUw4hktNTqJty0BQZqSlxGxX8LJuM+P8Y7tz7tHdIr5Y4Qeln50SW+K8fswArh8zIOK1fTpmxHw5A2cLGdx4xkB+8uzXDOnSJjQo1GlDOrFwU6AdoGOrNHaUVZFXWhkauiAoWL/+6KXHct4x3bgmzgU2ZsaaP57TYN9jM+Nf14xi6ZZiUpOTIg5gADedNYikJAuVRqPDrHvYBVXhY/xkpKXw/NWjGNqtDTn3fkL7jFQKy6tD7QE9vWsd0lMiS6y9oxoMw6+JyEhL4aVrA2cXFw7vwY+emcWEYV1JT0mmfUYqx/WJLe1Gt2Xcc95R3D5pIQs3FXHGEZ0Z1r0tf/b65wcPYted2p8vVuRz2aheoeqru757JMlJRnpK7PUg0V2Ie4ZV8QTbL04dFHsDnhG92jFn3U4qa+ro07EVi38XuO3yhGFdWbm9hA0F5bRITaZXx4xQt9vHLhseOmi+84uTmLlqB3/5rH74gB7tA4WeY3u24/Nl2zmia2smL97KiF7tSUlOihl47MQBHZm9toCnrhhJ7rqdXDyyB4M6t6akopp7vM4Evzx9IGeHFRxSkizUPtIyLZkT+2cx584zGPybyVx3an8mzdvItuLKUPXdxSN7MH1lPted2p9xR3XhhP4daZmaTFpKUkw7WsfMdDq2SqO0siZUH9+9XUs2FZYHzg6X59G9XUs2F5WHvksTxw8JNXrfNn4IOX060DJt9xfV7QsFfBPr3ynwxQ/vd763xgzuRPuM1FBYxis1jD+qyx6vdIX6Ufx2dwXnmMGdQgeZv/5wJIs2F5HdOj1URdO5TQtOG9KJc47uGtPVbtbtp1NVUxf3wrBou7vy8OSB2Zw8MP4dwIIBPHZIZ0oqamJKgFeN7kPHzDSO6Nom1GYRFDzzmXzjyWRlplNVU8ecdTt5Yda6mCBvSEP77qSBWREH53l3ndWo5Q3r0ZZ3fnESSzYX06tjBpnpKTEBP3pA/bJf//mJzFlXEPoetAorqQ7u3Jpl20ro3q4lMyaOZd2OXUyat5ERYQ19r19/IrPXFNAyLZlbxg2mttbxxrxNrMkv46jubUON3tEH1gGdWoe62gJcNbovT01bHXN2OqhzZPVI8Ltw3an9OGlAFhcM785JA7NDB7rgwGMdWqVRUFZFslnos4af/bVukUpachJVtXX86DuRXZs/u3kMD320jLcXbA79/6SnJLP0d+NIT0niq9U7AgHvfYfDrziHyK61b94wmvSUJKpq6kJX8c6580ycc7y3cAv//eI8ju3VjtLKGsYd2YWbzhxEz/YtadMylZx7P6GovJpxR3YJBfxFI3tELP+Acs41m38jR450h4vFm4rcV6vynXPOnfG/n7sHPljaxFsUsHhTkfvh379y5VU1e/3eZVuLXe9b33VjH5oSmlZTW+cen7LC9b71Xdf71ncP4JbGWr612C3cWJjQdezOy1+vd498vPygrOurVfnu7rcWNfr1f3hvibvhhTmuoLTSLd9avNfrK6usdjvLKt2WwnI39qEp7vK/z3QlFdW7fU9tbZ3bVlQed96uyhq3YluJG3j7++75mWt3u5wHJ3/ret/6rnt+xhp39p+nug0FZQ2+duX2EpdXUhF33tr8Utf71ndd7todMfN+/u9c1/vWd93Ls9fvdlv2pLqm1v3l0+Uur6TC7aqscXV1dRHz/z5tlet967uuorrmgP0mgFzXQKaaa0aX4Obk5Ljc3Nym3oyDrrSyhrTkpEaPoNdcbSjYxcl/msKATpl8ctOpEfP6THwP2LtxY6R5cgdwXJdVeaX0bJ+x2+9+VU0dHyzawnnHdEvYeDL3vruEp79cw+0ThnDtKf33/IYDYFVeKWWVNaE2rX1lZnOccznx5qmKphmI19hzKOrRviXXndp/t9395NB3IEM2vAdMQ9JSkhI+gNcNpw1gk9df/WBpzGffXyrBy0Hx3jdbyEhLDvUaEpEDY3cl+ITWCZjZODNbZmYrzWxiItclzds5R3dVuIscZIm86XYy8DgwHhgKXGZmQ3f/LhEROVASWYIfBax0zq12zlUB/wHOT+D6REQkTCIDvjsQPjjFRm9aBDO71sxyzSw3Ly8veraIiOyjJu+X55x7yjmX45zLyc6Of+GKiIjsvUQG/CYgfDCLHt40ERE5CBIZ8F8DA82sr5mlAZcCbydwfSIiEiZhV9g452rM7L+BD4Fk4B/Oudg7HYiISEIk9BJK59z7wPuJXIeIiMTXrK5kNbM8IP6NMfcsC8jf46sOL9on8Wm/xKf9EutQ2Ce9nXNxe6g0q4DfH2aW29Dluocr7ZP4tF/i036JdajvkybvJikiIomhgBcR8Sk/BfxTTb0BzZD2SXzaL/Fpv8Q6pPeJb+rgRUQkkp9K8CIiEkYBLyLiU4d8wB8ONxUxs3+Y2XYzWxQ2rYOZfWxmK7y/7b3pZmaPefvjGzMbEfaeK73XrzCzK8OmjzSzhd57HrNE3fjyADKznmY2xcyWmNliM/uVN/1w3y8tzGy2mS3w9ss93vS+ZjbL+ywve8OHYGbp3vOV3vw+Ycu6zZu+zMzODpt+SP7mzCzZzOaZ2bvec//vk4buxn0o/CMwBMIqoB+QBiwAhjb1diXgc54CjAAWhU37EzDRezwReMB7PAH4ADDgeGCWN70DsNr729573N6bN9t7rXnvHd/Un7kR+6QrMMJ73BpYTuDGMof7fjEg03ucCszyPsMrwKXe9CeBn3uPrwee9B5fCrzsPR7q/Z7Sgb7e7yz5UP7NATcBLwLves99v08O9RL8YXFTEefcNKAgavL5wD+9x/8ELgib/rwL+ApoZ2ZdgbOBj51zBc65ncDHwDhvXhvn3Fcu8C1+PmxZzZZzbotzbq73uARYSuB+A4f7fnHOuVLvaar3zwFjgde86dH7Jbi/XgNO985Uzgf+45yrdM6tAVYS+L0dkr85M+sBnAM87T03DoN9cqgHfKNuKuJTnZ1zW7zHW4HO3uOG9snupm+MM/2Q4Z1CDydQWj3s94tXFTEf2E7ggLUKKHTO1XgvCf8soc/vzS8COrL3+6u5ewS4BajznnfkMNgnh3rAC4FSG4FS2mHHzDKB14EbnXPF4fMO1/3inKt1zh1L4B4Mo4AhTbxJTcrMzgW2O+fmNPW2HGyHesAfzjcV2eZVI+D93e5Nb2if7G56jzjTmz0zSyUQ7i84597wJh/2+yXIOVcITAFOIFAlFRw9NvyzhD6/N78tsIO931/N2WjgPDNbS6D6ZCzwKIfDPmnqRoD9+UdguOPVBBo8go0bRzb1diXos/YhspH1QSIbE//kPT6HyMbE2d70DsAaAg2J7b3HHbx50Y2JE5r68zZifxiBevFHoqYf7vslG2jnPW4JfAGcC7xKZIPi9d7jG4hsUHzFe3wkkQ2Kqwk0Jh7SvzlgDPWNrL7fJ02+AQfgP2wCgR4Uq4A7mnp7EvQZXwK2ANUE6veuIVAn+CmwAvgkLJQMeNzbHwuBnLDlXE2gYWglcFXY9Bxgkfee/8O7wrk5/wNOIlD98g0w3/s3QfuFo4F53n5ZBNzlTe9H4IC10gu2dG96C+/5Sm9+v7Bl3eF99mWE9SA6lH9zUQHv+32ioQpERHzqUK+DFxGRBijgRUR8SgEvIuJTCngREZ9SwIuI+JQCXg5LZnaHN9riN2Y238y+Y2Y3mllGU2+byIGibpJy2DGzE4CHgTHOuUozyyJwgcoMAv3j85t0A0UOEJXg5XDUFch3zlUCeIF+MdANmGJmUwDM7Cwzm2lmc83sVW/cG8xsrZn9yRsrfraZDfCmX2Jmi7yx2Kc1zUcTqacSvBx2vKD+EsggcLXry865qd5YJTnOuXyvVP8GgasVy8zsVgJXOv7Oe93fnXP3mdmPge875841s4XAOOfcJjNr5wJjwYg0GZXg5bDjAuOljwSuBfKAl83sJ1EvO57ADR6me0PvXgn0Dpv/UtjfE7zH04HnzOxnBMYoEWlSKXt+iYj/OOdqgc+Bz72S95VRLzECNwK5rKFFRD92zl1nZt8hMLDZHDMb6ZzbcWC3XKTxVIKXw46ZDTazgWGTjgXWASUEbv8H8BUwOqx+vZWZDQp7zw/C/s70XtPfOTfLOXcXgTOD8CFkRQ46leDlcJQJ/MXM2gE1BEYNvBa4DJhsZpudc6d51TYvmVm6977fEBgxEKC9mX0DVHrvA3jQO3AYgREtFxyUTyPSADWyiuyl8MbYpt4Wkd1RFY2IiE+pBC8i4lMqwYuI+JQCXkTEpxTwIiI+pYAXEfEpBbyIiE/9f9VoZPKxnrrDAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nv3VHD585lc9"
      },
      "source": [
        "## Testing the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JD6M4tb8l2Vs",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "af223a67-042a-46e3-d049-d178fdf1d451"
      },
      "source": [
        "model.eval()\n",
        "input_ids = tokenizer.encode(\"WebNLG: sidharth | hometown | Delhi && sidharth | play |  football </s>\", return_tensors=\"pt\")  # Batch size 1\n",
        "input_ids=input_ids.to(dev)\n",
        "outputs = model.generate(input_ids)\n",
        "tokenizer.decode(outputs[0])"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<pad> Sidharth, who play football, is located in Delhi.</s>'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "owrlOe0L62WK"
      },
      "source": [
        "Before testing the model further, lets learn how to serialize it and load from the path"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NG5aC_Yi5aQR"
      },
      "source": [
        "## Serializing the trained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OZ1A6VdZtLTs"
      },
      "source": [
        "torch.save(model.state_dict(),'pytoch_model.bin')\n"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ea11_6C7Qth7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vKiXqlxqzZ7b"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "osi7Pi2ptSAq"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdUbFoxPGjR7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rxUEcuX_Gkkc"
      },
      "source": [
        "## Downloading the config file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UlhGuidlyM5v",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e574e512-92b9-410a-efd4-7578dffde3c5"
      },
      "source": [
        "!wget https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-config.json"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-04-10 19:41:07--  https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-config.json\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.133.5\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.133.5|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1199 (1.2K) [application/json]\n",
            "Saving to: ‘t5-base-config.json’\n",
            "\n",
            "\rt5-base-config.json   0%[                    ]       0  --.-KB/s               \rt5-base-config.json 100%[===================>]   1.17K  --.-KB/s    in 0s      \n",
            "\n",
            "2021-04-10 19:41:07 (67.4 MB/s) - ‘t5-base-config.json’ saved [1199/1199]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "owUdVXZ9GvPt"
      },
      "source": [
        ""
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f_2MGB0jGv7T"
      },
      "source": [
        "## Loading the trained model from the path"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ffh8_weNtSMN"
      },
      "source": [
        "model = T5ForConditionalGeneration.from_pretrained('pytoch_model.bin', return_dict=True,config='t5-base-config.json')\n"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PfpNp3Q8HZLu"
      },
      "source": [
        ""
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uXNP97trHZt9"
      },
      "source": [
        "## The Inference function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8bFXJDpQEJ-p"
      },
      "source": [
        "def generate(text):\n",
        "  model.eval()\n",
        "  input_ids = tokenizer.encode(\"WebNLG:{} </s>\".format(text), return_tensors=\"pt\")  # Batch size 1\n",
        "  # input_ids.to(dev)\n",
        "  s = time.time()\n",
        "  outputs = model.generate(input_ids)\n",
        "  gen_text=tokenizer.decode(outputs[0]).replace('<pad>','').replace('</s>','')\n",
        "  elapsed = time.time() - s\n",
        "  print('Generated in {} seconds'.format(str(elapsed)[:4]))\n",
        "\n",
        "  \n",
        "  return gen_text"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IIKmb1jR2XS3"
      },
      "source": [
        ""
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LlKSpuxcIP_5"
      },
      "source": [
        "# Now, Lets test it out !"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dlKdla7l8Gs9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "75be746e-52d7-41f7-aaf3-19448252a0c1"
      },
      "source": [
        "generate(' Russia | leader | Putin')"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 0.57 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Putin is the leader of Russia.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uThvhALg7m3s",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "6cf8c9c0-a65a-4985-8eba-2bbee1e88099"
      },
      "source": [
        "generate('Sidhath | profession | Doctor  && Sidharth | home_town |  Bombay')"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 1.19 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Sidhath, a doctor, lives in Bombay and is a doctor'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3y08dQ7y75VX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "790400dd-d012-466d-d629-522e41e95b5f"
      },
      "source": [
        "generate('Nie_Haisheng | birthDate | 1964-10-13  && Nie_Haisheng | occupation | Fighter_pilot ')"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 1.22 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Born on October 13th, 1964, Nie Haisheng was a fighter pilot.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DQkgoIHB9bOu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "e132f06c-884d-4c3a-ed3e-fd9549cf7d8d"
      },
      "source": [
        "generate('Bananaman | creator | Steve_Bright &&  Bananaman | broadcastedBy | BBC')"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 1.14 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Bananaman was created by Steve Bright and broadcasted by the BBC.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8X3LAar9ohw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "848688ca-2127-48d6-d6e2-bdf79038d253"
      },
      "source": [
        "generate('Bananaman | lastAired | \"1986-04-15\" && Bananaman | creator | Steve_Bright')"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 1.22 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Steve Bright created the character Bananaman which aired on 15th April 1986.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3QHRL7QG9olX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "627270ba-84c1-431f-eb23-1029d9f39b34"
      },
      "source": [
        "generate('Alan_B._Miller_Hall | currentTenants | Mason_School_of_Business && Alan_B._Miller_Hall | location | Williamsburg,_Virginia')"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 1.24 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' The Mason School of Business are the current tenants of Alan B Miller Hall, which is located in'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aQ9SahJsHKrB"
      },
      "source": [
        ""
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vT9-vCAXH8uw"
      },
      "source": [
        "## Results after quantization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H587bxDGzfVr"
      },
      "source": [
        "import torch.quantization\n",
        "import torch.nn as nn\n",
        "\n",
        "quantized_model = torch.quantization.quantize_dynamic(\n",
        "    model, {nn.Linear,nn.Dropout,nn.LayerNorm}, dtype=torch.qint8\n",
        ")\n"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cFK85QPHT4Vp"
      },
      "source": [
        "Lets check the difference in size of the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMuTHF8Kziqq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e423adb2-db08-49de-d1ca-0abdb43de703"
      },
      "source": [
        "def print_size_of_model(model):\n",
        "    torch.save(model.state_dict(), \"temp.p\")\n",
        "    print('Size (MB):', os.path.getsize(\"temp.p\")/1e6)\n",
        "    os.remove('temp.p')\n",
        "\n",
        "print_size_of_model(model)\n",
        "print_size_of_model(quantized_model)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Size (MB): 891.733816\n",
            "Size (MB): 322.031443\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_AcH6CQHKwy"
      },
      "source": [
        ""
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JKhWKMiABPmi"
      },
      "source": [
        "def quant_generate(text):\n",
        "  quantized_model.eval()\n",
        "  input_ids = tokenizer.encode(\"WebNLG:{} </s>\".format(text), return_tensors=\"pt\")  # Batch size 1\n",
        "  # input_ids.to(dev)\n",
        "  s = time.time()\n",
        "  outputs = quantized_model.generate(input_ids)\n",
        "  gen_text=tokenizer.decode(outputs[0]).replace('<pad>','').replace('</s>','')\n",
        "  elapsed = time.time() - s\n",
        "  print('Generated in {} seconds'.format(str(elapsed)[:4]))\n",
        "\n",
        "  \n",
        "  return gen_text"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4AYLOcvcHKvG"
      },
      "source": [
        ""
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HbJKHp9pT-2c"
      },
      "source": [
        "Now lets check the difference in inference time"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BtHtJ1PvHKll",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "72296750-6b4a-40de-931f-6651d81a75d0"
      },
      "source": [
        "quant_generate('Facebook | CEO | Mark  && Facebook | number Of Employees | 52000 ')"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 0.66 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' The CEO of Facebook is Mark and there are 52000 employees.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2YMla4UF-0en",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "9bca7c15-f8ee-4b9e-f690-c3274fa7bfb9"
      },
      "source": [
        "generate('Facebook | CEO | Mark  && Facebook | number Of Employees | 52000 ')"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated in 1.20 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' The CEO of the company, Facebook, has 52000 employees and is the CEO.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8lJItBxPHKjq"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}