Confidence_based_allocation_agnews.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5tUoHe9FRhs"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dr7BCHS-nIRW",
        "outputId": "da71ac4a-2059-4bab-9df7-145ac6436a3c",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m78.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m34.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m105.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m84.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import torch\n",
        "from torch.nn import BCEWithLogitsLoss, BCELoss\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.metrics import classification_report, confusion_matrix, multilabel_confusion_matrix, f1_score, accuracy_score\n",
        "import pickle\n",
        "#from transformers import *\n",
        "from tqdm import tqdm, trange\n",
        "from ast import literal_eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UhjnJEwKnISB",
        "outputId": "9889c969-a0cc-41f9-dd56-74eb62e72e41"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ]
        }
      ],
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "uorMX_zrnISM",
        "outputId": "86d2d872-a7b3-4056-e904-e6b36b1f3356",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla T4'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Exploration"
      ],
      "metadata": {
        "id": "NHcY8kii7Qot"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "62QeEJcKtWm3",
        "outputId": "d3471a94-5ec8-4293-cfb2-5fb7e5a3b1f1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  Class Index      conf  \\\n",
              "0  Title: Wall St. Bears Claw Back Into the Black...            3  0.905556   \n",
              "1  Title: Carlyle Looks Toward Commercial Aerospa...            3  0.922222   \n",
              "2  Title: Oil and Economy Cloud Stocks' Outlook (...            3  0.927778   \n",
              "3  Title: Iraq Halts Oil Exports from Main Southe...            3  0.911111   \n",
              "4  Title: Oil prices soar to all-time record, pos...            3  0.911111   \n",
              "\n",
              "   MV_world  MV_sports  MV_business  MV_sci  \n",
              "0         0          0            1       0  \n",
              "1         0          0            1       0  \n",
              "2         0          0            1       0  \n",
              "3         0          0            1       0  \n",
              "4         0          0            1       0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-eb0f0346-eb7d-4c88-8651-3d3c2abf872b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Class Index</th>\n",
              "      <th>conf</th>\n",
              "      <th>MV_world</th>\n",
              "      <th>MV_sports</th>\n",
              "      <th>MV_business</th>\n",
              "      <th>MV_sci</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Title: Wall St. Bears Claw Back Into the Black...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.905556</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Title: Carlyle Looks Toward Commercial Aerospa...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.922222</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Title: Oil and Economy Cloud Stocks' Outlook (...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.927778</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Title: Iraq Halts Oil Exports from Main Southe...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.911111</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Title: Oil prices soar to all-time record, pos...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.911111</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-eb0f0346-eb7d-4c88-8651-3d3c2abf872b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-eb0f0346-eb7d-4c88-8651-3d3c2abf872b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-eb0f0346-eb7d-4c88-8651-3d3c2abf872b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import pandas as pd\n",
        "data=pd.read_csv('agnews_train8.csv')\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data2=pd.read_csv('agnews_train2.csv')\n",
        "data2['conf']=data['conf']"
      ],
      "metadata": {
        "id": "R3m3ECwPGxEK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 353
        },
        "outputId": "fd7dd5fe-745f-498e-cc3d-24a67da0b5e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-fe68cccbdd57>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'agnews_train2.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdata2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'conf'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'conf'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    329\u001b[0m                     \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfind_stack_level\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m                 )\n\u001b[0;32m--> 331\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0;31m# error: \"Callable[[VarArg(Any), KwArg(Any)], Any]\" has no\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 950\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    951\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    603\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 605\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    606\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    607\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m   1440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1442\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1443\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1444\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m   1733\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1734\u001b[0m                     \u001b[0mmode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m\"b\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1735\u001b[0;31m             self.handles = get_handle(\n\u001b[0m\u001b[1;32m   1736\u001b[0m                 \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1737\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    854\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    855\u001b[0m             \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 856\u001b[0;31m             handle = open(\n\u001b[0m\u001b[1;32m    857\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    858\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'agnews_train2.csv'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def append(text):\n",
        "  return text+\" and give a confidence score for your prediction on a scale of 0 to 1.\"\n",
        "data2['prompt1']=data2['prompt1'].apply(append)\n",
        "data2['prompt2']=data2['prompt2'].apply(append)\n",
        "data2['prompt3']=data2['prompt3'].apply(append)\n",
        "data2['prompt4']=data2['prompt4'].apply(append)\n",
        "data2['prompt5']=data2['prompt5'].apply(append)\n",
        "data2['prompt6']=data2['prompt6'].apply(append)\n",
        "data2['prompt7']=data2['prompt7'].apply(append)\n",
        "data2['prompt8']=data2['prompt8'].apply(append)\n",
        "data2['prompt9']=data2['prompt9'].apply(append)\n",
        "data2['prompt10']=data2['prompt10'].apply(append)\n",
        "data2['prompt11']=data2['prompt11'].apply(append)\n",
        "data2['prompt12']=data2['prompt12'].apply(append)\n",
        "data2['prompt13']=data2['prompt13'].apply(append)"
      ],
      "metadata": {
        "id": "TSb3lAuRIIhw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#length calculation for cost computation\n",
        "df=data2\n",
        "df['length1']=df['prompt1'].apply(lambda x:len(x.split()))\n",
        "df['length2']=df['prompt2'].apply(lambda x:len(x.split()))\n",
        "df['length3']=df['prompt3'].apply(lambda x:len(x.split()))\n",
        "df['length4']=df['prompt4'].apply(lambda x:len(x.split()))\n",
        "df['length5']=df['prompt5'].apply(lambda x:len(x.split()))\n",
        "df['length6']=df['prompt6'].apply(lambda x:len(x.split()))\n",
        "df['length7']=df['prompt7'].apply(lambda x:len(x.split()))\n",
        "df['length8']=df['prompt8'].apply(lambda x:len(x.split()))\n",
        "df['length9']=df['prompt9'].apply(lambda x:len(x.split()))\n",
        "df['length10']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length11']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length12']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length13']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "\n",
        "df['length']=df['length1']+df['length2']+df['length3']+df['length4']+df['length5']+df['length6']+df['length7']+df['length8']+df['length9']+df['length10']+df['length11']+df['length12']+df['length13']\n",
        "\n",
        "df=df.sort_values(by='conf',ascending=False,ignore_index=True)\n",
        "# Assuming your dataframe is called 'df'\n",
        "total_rows = df.shape[0]\n",
        "num_rows_20_percent = int(total_rows * 1) #can change this to 0.2, 0.4, 0.6, 0.8\n",
        "print(1154-num_rows_20_percent)\n",
        "df['allocate'] = 0  # Initialize the new column with 0\n",
        "df.loc[0:num_rows_20_percent-1, 'allocate'] = 1  # Assign 1 to the first 20% of rows\n",
        "df2=df[df['allocate']==0]\n",
        "df1=df[df['allocate']==1]\n",
        "sum(df1['length'].tolist())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7rc6D4g0Gv_t",
        "outputId": "448e92a7-ac1e-46a0-aa46-6b10dd1204e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1214327"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_abnormal=data[data['conf']>1]\n",
        "data_abnormal"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        },
        "id": "16-xWCrHVj_n",
        "outputId": "cfa47775-093c-4843-aabe-fc8abc630e22"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [Title: Coming to The Rescue Description:, Class Index, conf, MV_world, MV_sports, MV_business, MV_sci]\n",
              "Index: []"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6893939e-4498-4604-8e8e-6b072a1de62a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title: Coming to The Rescue Description:</th>\n",
              "      <th>Class Index</th>\n",
              "      <th>conf</th>\n",
              "      <th>MV_world</th>\n",
              "      <th>MV_sports</th>\n",
              "      <th>MV_business</th>\n",
              "      <th>MV_sci</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6893939e-4498-4604-8e8e-6b072a1de62a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6893939e-4498-4604-8e8e-6b072a1de62a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6893939e-4498-4604-8e8e-6b072a1de62a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.hist(data['conf'], bins=15, align='left')\n",
        "\n",
        "# Add axis labels and a title\n",
        "plt.xlabel('Confidence Score')\n",
        "plt.ylabel('Frequency')\n",
        "plt.title('AG News')\n",
        "# Display the plot\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "fEYeBp_u5E11",
        "outputId": "15da8c64-91cf-473e-e27e-a25ac26c57d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj8AAAHHCAYAAABQhTneAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/w0lEQVR4nO3de1yUdf7//+cAgqjMICoghYdMU1IzNXUqK5NCRdO00iIPZdkatqVm6nawrDx1MstDW67aN1zLXe2gqRFaWuIJD7lqng1LBmwNRm1FDu/fH/6cTxNWSAODXo/77Xbdbjvv631d79eba2ueveeaa2zGGCMAAACLCPB3AQAAABWJ8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AOg3MyYMUM2m03t27f/3X45OTkaM2aMWrRooRo1aqhq1aq6/PLLdd999+mrr776w3EOHTokm80mm82mf//73yX2P/vss7LZbPrxxx/LPBcAFw/CD4Byk5KSogYNGmjDhg3at2/fOfts2LBBV155paZOnao2bdpo8uTJevPNN9W3b19t2LBBHTt21OrVq0s95vjx48VPFgL4PUH+LgDAxengwYNau3atFi1apIceekgpKSkaN26cV5+ffvpJvXr1UlBQkLZu3aqmTZt67X/hhRe0YMEChYaGlmrMVq1aaevWrVq8eLF69+7ts7kAuLiw8gOgXKSkpKhmzZpKTEzUHXfcoZSUlBJ9Zs2apaysLE2dOrVE8JEkm82mu+++W9dcc02pxuzXr5+aNGlS6tWf9evXq0uXLnI4HKpWrZpuvPFGff31157933zzjWw2mz7++GNPW0ZGhmw2m1q3bu11rq5du3p9vLdp0yYlJCSodu3aCg0NVcOGDXX//feXah4AyhfhB0C5SElJUe/evRUcHKy7775be/fu1caNG736fPLJJwoNDfXZKk1gYKCeeuopbdu2TYsXL/7dvitXrtQNN9wgt9utcePGacKECcrNzdXNN9+sDRs2SJKaN2+u8PBwr4/d1qxZo4CAAG3btk1ut1uSVFxcrLVr1+qGG26QdOYepltvvVWHDh3SmDFj9MYbbygpKUnr1q3zyTwB/EkGAHxs06ZNRpJJTU01xhhTXFxsLr30UvPoo4969atZs6Zp1apViePdbrc5evSoZztx4sTvjnfw4EEjybz00kumsLDQNG7c2Fx11VWmuLjYGGPMuHHjjCRz9OhRTz2NGzc2CQkJnj7GGPPzzz+bhg0bmltuucXTlpiYaNq1a+d53bt3b9O7d28TGBholi1bZowxZvPmzUaS+eijj4wxxixevNhIMhs3biztnwxABWLlB4DPpaSkKCoqSp06dZJ05uOrvn37asGCBSoqKvL0c7vdqlGjRonj+/fvrzp16ni20aNHl3rsX67+fPjhh+fss3XrVu3du1f33HOP/vvf/+rHH3/Ujz/+qJMnT6pz585avXq1iouLJUkdO3bU5s2bdfLkSUnSV199pW7duqlVq1Zas2aNpDOrQTabTddff70kKTw8XJK0ZMkSFRQUlLp2ABWD8APAp4qKirRgwQJ16tRJBw8e1L59+7Rv3z61b99e2dnZSktL8/QNCwvTiRMnSpxj/PjxSk1NVWpqaplqSEpK0uWXX/6b9/7s3btXkjRw4ECvkFWnTh298847ys/PV15enqQz4aewsFDp6enavXu3cnJy1LFjR91www1e4ScuLk4RERGSpBtvvFF9+vTRc889p9q1a6tnz56aM2eO8vPzyzQfAL7Ft70A+NTKlSuVlZWlBQsWaMGCBSX2p6Sk6NZbb5UkNW3aVNu2bVNBQYGqVKni6dOyZcs/VcPZ1Z9Bgwbpo48+KrH/7KrOSy+9pFatWp3zHGdXpNq2bauqVatq9erVqlevniIjI9WkSRN17NhRM2bMUH5+vtasWaPbb7/dc6zNZtO//vUvrVu3Tp988olWrFih+++/X6+88orWrVt3ztUuABWH8APAp1JSUhQZGanp06eX2Ldo0SItXrxYs2bNUmhoqLp3765169Zp8eLFuuuuu3xax7333qsXXnhBzz33nG677TavfY0aNZIk2e12xcfH/+55goOD1a5dO61Zs0b16tVTx44dJZ1ZEcrPz1dKSoqys7M9Nzv/UocOHdShQwe9+OKLmj9/vpKSkrRgwQI98MADPpolgLLgYy8APvO///1PixYtUvfu3XXHHXeU2IYNG6bjx497vjo+dOhQRUVFafjw4dqzZ0+J853rI6vSOrv6s3XrVq+vqktSmzZt1KhRI7388svn/Njt6NGjXq87duyo9evXa9WqVZ7wU7t2bTVr1kyTJ0/29Dnrp59+KlH72RUmPvoC/I+VHwA+8/HHH+v48eMlVlrO6tChg+rUqaOUlBT17dtXERERWrx4sXr06KGrrrpK/fr10zXXXKMqVaro8OHDWrhwoSSpXr16ZaonKSlJzz//vLZu3erVHhAQoHfeeUddu3bVlVdeqfvuu0+XXHKJfvjhB61atUp2u12ffPKJp3/Hjh314osv6vDhw14h54YbbtBbb72lBg0a6NJLL/W0z5s3TzNmzNDtt9+uRo0a6fjx43r77bdlt9vVrVu3Ms0FgA/5+dtmAC4iPXr0MFWrVjUnT578zT6DBg0yVapUMT/++KOnLSsry4waNcrExcWZ0NBQExISYi677DIzYMAAs3r16j8c95dfdf+1OXPmGEleX3U/a8uWLaZ3796mVq1aJiQkxNSvX9/cddddJi0tzauf2+02gYGBJiwszBQWFnra33vvPSPJ9O/f36v/5s2bzd13323q1atnQkJCTGRkpOnevbvZtGnTH84FQPmzGcOP4AAAAOvgnh8AAGAphB8AAGAphB8AAGAphB8AAGAphB8AAGAphB8AAGApPORQZ37n58iRIwoLC5PNZvN3OQAAoBSMMTp+/LhiYmIUEFD69RzCj6QjR44oNjbW32UAAIAyOHz4sNdT1v8I4UdSWFiYpDN/PLvd7udqAABAabjdbsXGxnrex0vLr+GnqKhIzz77rN577z25XC7FxMRo0KBBeuqppzwfPxljNG7cOL399tvKzc3Vddddp5kzZ6px48ae8xw7dkyPPPKIPvnkEwUEBKhPnz56/fXXVaNGjVLVcXYsu91O+AEA4AJzvres+PWG58mTJ2vmzJl68803tWvXLk2ePFlTpkzRG2+84ekzZcoUTZs2TbNmzdL69etVvXp1JSQk6NSpU54+SUlJ2rFjh1JTU7VkyRKtXr1aQ4YM8ceUAABAJefX3/bq3r27oqKiNHv2bE9bnz59FBoaqvfee0/GGMXExGjkyJF6/PHHJUl5eXmKiorS3Llz1a9fP+3atUtxcXHauHGj2rZtK0lavny5unXrpu+//14xMTF/WIfb7ZbD4VBeXh4rPwAAXCDK+v7t15Wfa6+9VmlpadqzZ48kadu2bfrqq6/UtWtXSdLBgwflcrkUHx/vOcbhcKh9+/ZKT0+XJKWnpys8PNwTfCQpPj5eAQEBWr9+/TnHzc/Pl9vt9toAAIA1+PWenzFjxsjtdqtp06YKDAxUUVGRXnzxRSUlJUmSXC6XJCkqKsrruKioKM8+l8ulyMhIr/1BQUGKiIjw9Pm1iRMn6rnnnvP1dAAAwAXArys/H3zwgVJSUjR//nxt3rxZ8+bN08svv6x58+aV67hjx45VXl6eZzt8+HC5jgcAACoPv678jBo1SmPGjFG/fv0kSS1atNB3332niRMnauDAgYqOjpYkZWdnq27dup7jsrOz1apVK0lSdHS0cnJyvM5bWFioY8eOeY7/tZCQEIWEhJTDjAAAQGXn15Wfn3/+ucQTGQMDA1VcXCxJatiwoaKjo5WWlubZ73a7tX79ejmdTkmS0+lUbm6uMjIyPH1Wrlyp4uJitW/fvgJmAQAALiR+Xfnp0aOHXnzxRdWrV09XXnmltmzZoldffVX333+/pDPf23/sscf0wgsvqHHjxmrYsKGefvppxcTEqFevXpKkZs2aqUuXLnrwwQc1a9YsFRQUaNiwYerXr1+pvukFAACsxa/h54033tDTTz+thx9+WDk5OYqJidFDDz2kZ555xtPniSee0MmTJzVkyBDl5ubq+uuv1/Lly1W1alVPn5SUFA0bNkydO3f2PORw2rRp/pgSAACo5Pz6nJ/Kguf8AABw4bkgn/MDAABQ0Qg/AADAUgg/AADAUgg/AADAUgg/AADAUvz6VXcAAKyowZilfhn30KREv4xb2bDyAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALIXwAwAALMWv4adBgway2WwltuTkZEnSqVOnlJycrFq1aqlGjRrq06ePsrOzvc6RmZmpxMREVatWTZGRkRo1apQKCwv9MR0AAHAB8Gv42bhxo7KysjxbamqqJOnOO++UJA0fPlyffPKJFi5cqC+//FJHjhxR7969PccXFRUpMTFRp0+f1tq1azVv3jzNnTtXzzzzjF/mAwAAKj+bMcb4u4izHnvsMS1ZskR79+6V2+1WnTp1NH/+fN1xxx2SpG+//VbNmjVTenq6OnTooGXLlql79+46cuSIoqKiJEmzZs3S6NGjdfToUQUHB5dqXLfbLYfDoby8PNnt9nKbHwAAktRgzFK/jHtoUqJfxi0vZX3/rjT3/Jw+fVrvvfee7r//ftlsNmVkZKigoEDx8fGePk2bNlW9evWUnp4uSUpPT1eLFi08wUeSEhIS5Ha7tWPHjt8cKz8/X26322sDAADWUGnCz4cffqjc3FwNGjRIkuRyuRQcHKzw8HCvflFRUXK5XJ4+vww+Z/ef3fdbJk6cKIfD4dliY2N9NxEAAFCpVZrwM3v2bHXt2lUxMTHlPtbYsWOVl5fn2Q4fPlzuYwIAgMohyN8FSNJ3332nzz//XIsWLfK0RUdH6/Tp08rNzfVa/cnOzlZ0dLSnz4YNG7zOdfbbYGf7nEtISIhCQkJ8OAMAAHChqBQrP3PmzFFkZKQSE//vRqw2bdqoSpUqSktL87Tt3r1bmZmZcjqdkiSn06nt27crJyfH0yc1NVV2u11xcXEVNwEAAHDB8PvKT3FxsebMmaOBAwcqKOj/ynE4HBo8eLBGjBihiIgI2e12PfLII3I6nerQoYMk6dZbb1VcXJz69++vKVOmyOVy6amnnlJycjIrOwAA4Jz8Hn4+//xzZWZm6v777y+x77XXXlNAQID69Omj/Px8JSQkaMaMGZ79gYGBWrJkiYYOHSqn06nq1atr4MCBGj9+fEVOAQAAXEAq1XN+/IXn/AAAKhLP+fGNC/45PwAAABWB8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACyF8AMAACzF7+Hnhx9+0L333qtatWopNDRULVq00KZNmzz7jTF65plnVLduXYWGhio+Pl579+71OsexY8eUlJQku92u8PBwDR48WCdOnKjoqQAAgAuAX8PPTz/9pOuuu05VqlTRsmXLtHPnTr3yyiuqWbOmp8+UKVM0bdo0zZo1S+vXr1f16tWVkJCgU6dOefokJSVpx44dSk1N1ZIlS7R69WoNGTLEH1MCAACVnM0YY/w1+JgxY/T1119rzZo159xvjFFMTIxGjhypxx9/XJKUl5enqKgozZ07V/369dOuXbsUFxenjRs3qm3btpKk5cuXq1u3bvr+++8VExPzh3W43W45HA7l5eXJbrf7boIAAJxDgzFL/TLuoUmJfhm3vJT1/duvKz8ff/yx2rZtqzvvvFORkZG6+uqr9fbbb3v2Hzx4UC6XS/Hx8Z42h8Oh9u3bKz09XZKUnp6u8PBwT/CRpPj4eAUEBGj9+vXnHDc/P19ut9trAwAA1uDX8HPgwAHNnDlTjRs31ooVKzR06FD99a9/1bx58yRJLpdLkhQVFeV1XFRUlGefy+VSZGSk1/6goCBFRER4+vzaxIkT5XA4PFtsbKyvpwYAACopv4af4uJitW7dWhMmTNDVV1+tIUOG6MEHH9SsWbPKddyxY8cqLy/Psx0+fLhcxwMAAJWHX8NP3bp1FRcX59XWrFkzZWZmSpKio6MlSdnZ2V59srOzPfuio6OVk5Pjtb+wsFDHjh3z9Pm1kJAQ2e12rw0AAFiDX8PPddddp927d3u17dmzR/Xr15ckNWzYUNHR0UpLS/Psd7vdWr9+vZxOpyTJ6XQqNzdXGRkZnj4rV65UcXGx2rdvXwGzAAAAF5Igfw4+fPhwXXvttZowYYLuuusubdiwQX//+9/197//XZJks9n02GOP6YUXXlDjxo3VsGFDPf3004qJiVGvXr0knVkp6tKli+fjsoKCAg0bNkz9+vUr1Te9AACAtfg1/FxzzTVavHixxo4dq/Hjx6thw4aaOnWqkpKSPH2eeOIJnTx5UkOGDFFubq6uv/56LV++XFWrVvX0SUlJ0bBhw9S5c2cFBASoT58+mjZtmj+mBAAAKjm/PuensuA5PwCAisRzfnzjgnzODwAAQEUj/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEsh/AAAAEvxa/h59tlnZbPZvLamTZt69p86dUrJycmqVauWatSooT59+ig7O9vrHJmZmUpMTFS1atUUGRmpUaNGqbCwsKKnAgAALhBB/i7gyiuv1Oeff+55HRT0fyUNHz5cS5cu1cKFC+VwODRs2DD17t1bX3/9tSSpqKhIiYmJio6O1tq1a5WVlaUBAwaoSpUqmjBhQoXPBQAAVH5+Dz9BQUGKjo4u0Z6Xl6fZs2dr/vz5uvnmmyVJc+bMUbNmzbRu3Tp16NBBn332mXbu3KnPP/9cUVFRatWqlZ5//nmNHj1azz77rIKDgyt6OgAAoJLz+z0/e/fuVUxMjC677DIlJSUpMzNTkpSRkaGCggLFx8d7+jZt2lT16tVTenq6JCk9PV0tWrRQVFSUp09CQoLcbrd27Njxm2Pm5+fL7XZ7bQAAwBr8Gn7at2+vuXPnavny5Zo5c6YOHjyojh076vjx43K5XAoODlZ4eLjXMVFRUXK5XJIkl8vlFXzO7j+777dMnDhRDofDs8XGxvp2YgAAoNLy68deXbt29fzvli1bqn379qpfv74++OADhYaGltu4Y8eO1YgRIzyv3W43AQgAAIvw+8devxQeHq4mTZpo3759io6O1unTp5Wbm+vVJzs723OPUHR0dIlvf519fa77iM4KCQmR3W732gAAgDVUqvBz4sQJ7d+/X3Xr1lWbNm1UpUoVpaWlefbv3r1bmZmZcjqdkiSn06nt27crJyfH0yc1NVV2u11xcXEVXj8AAKj8/Pqx1+OPP64ePXqofv36OnLkiMaNG6fAwEDdfffdcjgcGjx4sEaMGKGIiAjZ7XY98sgjcjqd6tChgyTp1ltvVVxcnPr3768pU6bI5XLpqaeeUnJyskJCQvw5NQAAUEn5Nfx8//33uvvuu/Xf//5XderU0fXXX69169apTp06kqTXXntNAQEB6tOnj/Lz85WQkKAZM2Z4jg8MDNSSJUs0dOhQOZ1OVa9eXQMHDtT48eP9NSUAAFDJ2Ywxxt9F+Jvb7ZbD4VBeXh73/wAAyl2DMUv9Mu6hSYl+Gbe8lPX9u1Ld8wMAAFDeCD8AAMBSCD8AAMBSyhR+Dhw44Os6AAAAKkSZws/ll1+uTp066b333tOpU6d8XRMAAEC5KVP42bx5s1q2bKkRI0YoOjpaDz30kDZs2ODr2gAAAHyuTOGnVatWev3113XkyBH94x//UFZWlq6//no1b95cr776qo4ePerrOgEAAHziT93wHBQUpN69e2vhwoWaPHmy9u3bp8cff1yxsbEaMGCAsrKyfFUnAACAT/yp8LNp0yY9/PDDqlu3rl599VU9/vjj2r9/v1JTU3XkyBH17NnTV3UCAAD4RJl+3uLVV1/VnDlztHv3bnXr1k3vvvuuunXrpoCAM1mqYcOGmjt3rho0aODLWgEAAP60MoWfmTNn6v7779egQYNUt27dc/aJjIzU7Nmz/1RxAAAAvlam8LN3794/7BMcHKyBAweW5fQAAADlpkz3/MyZM0cLFy4s0b5w4ULNmzfvTxcFAABQXsoUfiZOnKjatWuXaI+MjNSECRP+dFEAAADlpUzhJzMzUw0bNizRXr9+fWVmZv7pogAAAMpLmcJPZGSkvvnmmxLt27ZtU61atf50UQAAAOWlTOHn7rvv1l//+letWrVKRUVFKioq0sqVK/Xoo4+qX79+vq4RAADAZ8r0ba/nn39ehw4dUufOnRUUdOYUxcXFGjBgAPf8AACASq1M4Sc4OFjvv/++nn/+eW3btk2hoaFq0aKF6tev7+v6AAAAfKpM4eesJk2aqEmTJr6qBQAAoNyVKfwUFRVp7ty5SktLU05OjoqLi732r1y50ifFAQAA+FqZws+jjz6quXPnKjExUc2bN5fNZvN1XQAAAOWiTOFnwYIF+uCDD9StWzdf1wMAAFCuyvRV9+DgYF1++eW+rgUAAKDclSn8jBw5Uq+//rqMMb6uBwAAoFyV6WOvr776SqtWrdKyZct05ZVXqkqVKl77Fy1a5JPiAAAAfK1M4Sc8PFy33367r2sBAAAod2UKP3PmzPF1HQAAABWiTPf8SFJhYaE+//xzvfXWWzp+/Lgk6ciRIzpx4oTPigMAAPC1Mq38fPfdd+rSpYsyMzOVn5+vW265RWFhYZo8ebLy8/M1a9YsX9cJAADgE2Va+Xn00UfVtm1b/fTTTwoNDfW033777UpLS/NZcQAAAL5WppWfNWvWaO3atQoODvZqb9CggX744QefFAYAAFAeyrTyU1xcrKKiohLt33//vcLCwv50UQAAAOWlTOHn1ltv1dSpUz2vbTabTpw4oXHjxvGTFwAAoFIr08der7zyihISEhQXF6dTp07pnnvu0d69e1W7dm3985//9HWNAAAAPlOmlZ9LL71U27Zt09/+9jcNHz5cV199tSZNmqQtW7YoMjKyTIVMmjRJNptNjz32mKft1KlTSk5OVq1atVSjRg316dNH2dnZXsdlZmYqMTFR1apVU2RkpEaNGqXCwsIy1QAAAC5+ZVr5kaSgoCDde++9Pili48aNeuutt9SyZUuv9uHDh2vp0qVauHChHA6Hhg0bpt69e+vrr7+WJBUVFSkxMVHR0dFau3atsrKyNGDAAFWpUkUTJkzwSW0AAODiUqbw8+677/7u/gEDBpT6XCdOnFBSUpLefvttvfDCC572vLw8zZ49W/Pnz9fNN98s6cyTpZs1a6Z169apQ4cO+uyzz7Rz5059/vnnioqKUqtWrfT8889r9OjRevbZZ0t8Gw0AAKBM4efRRx/1el1QUKCff/5ZwcHBqlat2nmFn+TkZCUmJio+Pt4r/GRkZKigoEDx8fGetqZNm6pevXpKT09Xhw4dlJ6erhYtWigqKsrTJyEhQUOHDtWOHTt09dVXn3PM/Px85efne1673e5S1wsAAC5sZQo/P/30U4m2vXv3aujQoRo1alSpz7NgwQJt3rxZGzduLLHP5XIpODhY4eHhXu1RUVFyuVyePr8MPmf3n933WyZOnKjnnnuu1HUCAICLR5l/2+vXGjdurEmTJpVYFfothw8f1qOPPqqUlBRVrVrVV2WUytixY5WXl+fZDh8+XKHjAwAA//FZ+JHO3AR95MiRUvXNyMhQTk6OWrduraCgIAUFBenLL7/UtGnTFBQUpKioKJ0+fVq5ublex2VnZys6OlqSFB0dXeLbX2dfn+1zLiEhIbLb7V4bAACwhjJ97PXxxx97vTbGKCsrS2+++aauu+66Up2jc+fO2r59u1fbfffdp6ZNm2r06NGKjY1VlSpVlJaWpj59+kiSdu/erczMTDmdTkmS0+nUiy++qJycHM9X7FNTU2W32xUXF1eWqQEAgItcmcJPr169vF7bbDbVqVNHN998s1555ZVSnSMsLEzNmzf3aqtevbpq1arlaR88eLBGjBihiIgI2e12PfLII3I6nerQoYOkM0+ajouLU//+/TVlyhS5XC499dRTSk5OVkhISFmmBgAALnJlCj/FxcW+ruOcXnvtNQUEBKhPnz7Kz89XQkKCZsyY4dkfGBioJUuWaOjQoXI6napevboGDhyo8ePHV0h9AADgwmMzxhh/F+FvbrdbDodDeXl53P8DACh3DcYs9cu4hyYl+mXc8lLW9+8yrfyMGDGi1H1fffXVsgwBAABQLsoUfrZs2aItW7aooKBAV1xxhSRpz549CgwMVOvWrT39bDabb6oEAADwkTKFnx49eigsLEzz5s1TzZo1JZ158OF9992njh07auTIkT4tEgAAwFfK9JyfV155RRMnTvQEH0mqWbOmXnjhhVJ/2wsAAMAfyhR+3G63jh49WqL96NGjOn78+J8uCgAAoLyUKfzcfvvtuu+++7Ro0SJ9//33+v777/Xvf/9bgwcPVu/evX1dIwAAgM+U6Z6fWbNm6fHHH9c999yjgoKCMycKCtLgwYP10ksv+bRAAAAAXypT+KlWrZpmzJihl156Sfv375ckNWrUSNWrV/dpcQAAAL72p37YNCsrS1lZWWrcuLGqV68unpcIAAAquzKFn//+97/q3LmzmjRpom7duikrK0vSmd/i4mvuAACgMitT+Bk+fLiqVKmizMxMVatWzdPet29fLV++3GfFAQAA+FqZ7vn57LPPtGLFCl166aVe7Y0bN9Z3333nk8IAAADKQ5lWfk6ePOm14nPWsWPHFBIS8qeLAgAAKC9lCj8dO3bUu+++63lts9lUXFysKVOmqFOnTj4rDgAAwNfK9LHXlClT1LlzZ23atEmnT5/WE088oR07dujYsWP6+uuvfV0jAACAz5Rp5ad58+bas2ePrr/+evXs2VMnT55U7969tWXLFjVq1MjXNQIAAPjMea/8FBQUqEuXLpo1a5aefPLJ8qgJAACg3Jz3yk+VKlX0zTfflEctAAAA5a5MH3vde++9mj17tq9rAQAAKHdluuG5sLBQ//jHP/T555+rTZs2JX7T69VXX/VJcQAAAL52XuHnwIEDatCggf7zn/+odevWkqQ9e/Z49bHZbL6rDgAAwMfOK/w0btxYWVlZWrVqlaQzP2cxbdo0RUVFlUtxAAAAvnZe9/z8+lfbly1bppMnT/q0IAAAgPJUphuez/p1GAIAAKjsziv82Gy2Evf0cI8PAAC4kJzXPT/GGA0aNMjz46WnTp3SX/7ylxLf9lq0aJHvKgQAAPCh8wo/AwcO9Hp97733+rQYAACA8nZe4WfOnDnlVQcAAECF+FM3PAMAAFxoCD8AAMBSCD8AAMBSCD8AAMBSCD8AAMBSCD8AAMBS/Bp+Zs6cqZYtW8put8tut8vpdGrZsmWe/adOnVJycrJq1aqlGjVqqE+fPsrOzvY6R2ZmphITE1WtWjVFRkZq1KhRKiwsrOipAACAC4Rfw8+ll16qSZMmKSMjQ5s2bdLNN9+snj17aseOHZKk4cOH65NPPtHChQv15Zdf6siRI+rdu7fn+KKiIiUmJur06dNau3at5s2bp7lz5+qZZ57x15QAAEAlZzOV7NdJIyIi9NJLL+mOO+5QnTp1NH/+fN1xxx2SpG+//VbNmjVTenq6OnTooGXLlql79+46cuSIoqKiJEmzZs3S6NGjdfToUQUHB5dqTLfbLYfDoby8PNnt9nKbGwAAktRgzFK/jHtoUqJfxi0vZX3/rjT3/BQVFWnBggU6efKknE6nMjIyVFBQoPj4eE+fpk2bql69ekpPT5ckpaenq0WLFp7gI0kJCQlyu92e1aNzyc/Pl9vt9toAAIA1+D38bN++XTVq1FBISIj+8pe/aPHixYqLi5PL5VJwcLDCw8O9+kdFRcnlckmSXC6XV/A5u//svt8yceJEORwOzxYbG+vbSQEAgErL7+Hniiuu0NatW7V+/XoNHTpUAwcO1M6dO8t1zLFjxyovL8+zHT58uFzHAwAAlcd5/bBpeQgODtbll18uSWrTpo02btyo119/XX379tXp06eVm5vrtfqTnZ2t6OhoSVJ0dLQ2bNjgdb6z3wY72+dcQkJCFBIS4uOZAACAC4HfV35+rbi4WPn5+WrTpo2qVKmitLQ0z77du3crMzNTTqdTkuR0OrV9+3bl5OR4+qSmpsputysuLq7CawcAAJWfX1d+xo4dq65du6pevXo6fvy45s+fry+++EIrVqyQw+HQ4MGDNWLECEVERMhut+uRRx6R0+lUhw4dJEm33nqr4uLi1L9/f02ZMkUul0tPPfWUkpOTWdkBAADn5Nfwk5OTowEDBigrK0sOh0MtW7bUihUrdMstt0iSXnvtNQUEBKhPnz7Kz89XQkKCZsyY4Tk+MDBQS5Ys0dChQ+V0OlW9enUNHDhQ48eP99eUAABAJVfpnvPjDzznBwBQkXjOj29c8M/5AQAAqAiEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCl+DT8TJ07UNddco7CwMEVGRqpXr17avXu3V59Tp04pOTlZtWrVUo0aNdSnTx9lZ2d79cnMzFRiYqKqVaumyMhIjRo1SoWFhRU5FQAAcIHwa/j58ssvlZycrHXr1ik1NVUFBQW69dZbdfLkSU+f4cOH65NPPtHChQv15Zdf6siRI+rdu7dnf1FRkRITE3X69GmtXbtW8+bN09y5c/XMM8/4Y0oAAKCSsxljjL+LOOvo0aOKjIzUl19+qRtuuEF5eXmqU6eO5s+frzvuuEOS9O2336pZs2ZKT09Xhw4dtGzZMnXv3l1HjhxRVFSUJGnWrFkaPXq0jh49quDg4D8c1+12y+FwKC8vT3a7vVznCABAgzFL/TLuoUmJfhm3vJT1/btS3fOTl5cnSYqIiJAkZWRkqKCgQPHx8Z4+TZs2Vb169ZSeni5JSk9PV4sWLTzBR5ISEhLkdru1Y8eOCqweAABcCIL8XcBZxcXFeuyxx3TdddepefPmkiSXy6Xg4GCFh4d79Y2KipLL5fL0+WXwObv/7L5zyc/PV35+vue12+321TQAAEAlV2lWfpKTk/Wf//xHCxYsKPexJk6cKIfD4dliY2PLfUwAAFA5VIrwM2zYMC1ZskSrVq3SpZde6mmPjo7W6dOnlZub69U/Oztb0dHRnj6//vbX2ddn+/za2LFjlZeX59kOHz7sw9kAAIDKzK/hxxijYcOGafHixVq5cqUaNmzotb9NmzaqUqWK0tLSPG27d+9WZmamnE6nJMnpdGr79u3Kycnx9ElNTZXdbldcXNw5xw0JCZHdbvfaAACANfj1np/k5GTNnz9fH330kcLCwjz36DgcDoWGhsrhcGjw4MEaMWKEIiIiZLfb9cgjj8jpdKpDhw6SpFtvvVVxcXHq37+/pkyZIpfLpaeeekrJyckKCQnx5/QAAEAl5NfwM3PmTEnSTTfd5NU+Z84cDRo0SJL02muvKSAgQH369FF+fr4SEhI0Y8YMT9/AwEAtWbJEQ4cOldPpVPXq1TVw4ECNHz++oqYBAAAuIJXqOT/+wnN+AAAVief8+MZF8ZwfAACA8kb4AQAAlkL4AQAAllJpnvAMAEBF89e9N/AvVn4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAIClEH4AAICl+DX8rF69Wj169FBMTIxsNps+/PBDr/3GGD3zzDOqW7euQkNDFR8fr71793r1OXbsmJKSkmS32xUeHq7BgwfrxIkTFTgLAABwIfFr+Dl58qSuuuoqTZ8+/Zz7p0yZomnTpmnWrFlav369qlevroSEBJ06dcrTJykpSTt27FBqaqqWLFmi1atXa8iQIRU1BQAAcIEJ8ufgXbt2VdeuXc+5zxijqVOn6qmnnlLPnj0lSe+++66ioqL04Ycfql+/ftq1a5eWL1+ujRs3qm3btpKkN954Q926ddPLL7+smJiYCpsLAAC4MFTae34OHjwol8ul+Ph4T5vD4VD79u2Vnp4uSUpPT1d4eLgn+EhSfHy8AgICtH79+t88d35+vtxut9cGAACsodKGH5fLJUmKioryao+KivLsc7lcioyM9NofFBSkiIgIT59zmThxohwOh2eLjY31cfUAAKCy8uvHXv4yduxYjRgxwvPa7XYTgAAAF70GY5b6ZdxDkxL9Mu5vqbQrP9HR0ZKk7Oxsr/bs7GzPvujoaOXk5HjtLyws1LFjxzx9ziUkJER2u91rAwAA1lBpw0/Dhg0VHR2ttLQ0T5vb7db69evldDolSU6nU7m5ucrIyPD0WblypYqLi9W+ffsKrxkAAFR+fv3Y68SJE9q3b5/n9cGDB7V161ZFRESoXr16euyxx/TCCy+ocePGatiwoZ5++mnFxMSoV69ekqRmzZqpS5cuevDBBzVr1iwVFBRo2LBh6tevH9/0AgAA5+TX8LNp0yZ16tTJ8/rsfTgDBw7U3Llz9cQTT+jkyZMaMmSIcnNzdf3112v58uWqWrWq55iUlBQNGzZMnTt3VkBAgPr06aNp06ZV+FwAAMCFwWaMMf4uwt/cbrccDofy8vK4/wcALMRfNwBbTXnd8FzW9+9Ke88PAABAeSD8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASyH8AAAASwnydwEAADQYs9TfJcBCWPkBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWQvgBAACWEuTvAnxl+vTpeumll+RyuXTVVVfpjTfeULt27fxdFgBcUBqMWervEoByd1GEn/fff18jRozQrFmz1L59e02dOlUJCQnavXu3IiMj/V0egAsYYQC4+NiMMcbfRfxZ7du31zXXXKM333xTklRcXKzY2Fg98sgjGjNmzB8e73a75XA4lJeXJ7vdXt7lVgh//Qv70KREv4xrtfn6E2EAwPkqr39XlvX9+4Jf+Tl9+rQyMjI0duxYT1tAQIDi4+OVnp7ux8qsiTdGAEBld8GHnx9//FFFRUWKioryao+KitK33357zmPy8/OVn5/veZ2XlyfpTIL0tebjVvj8nKg86g1f6O8SAKDSK4/311+e93w/xLrgw09ZTJw4Uc8991yJ9tjYWD9UAwDAxc0xtXzPf/z4cTkcjlL3v+DDT+3atRUYGKjs7Gyv9uzsbEVHR5/zmLFjx2rEiBGe18XFxTp27Jhq1aolm81WrvVeiNxut2JjY3X48OGL5p6oiwnXp3Lj+lRuXJ/K7Y+ujzFGx48fV0xMzHmd94IPP8HBwWrTpo3S0tLUq1cvSWfCTFpamoYNG3bOY0JCQhQSEuLVFh4eXs6VXvjsdjv/cqjEuD6VG9encuP6VG6/d33OZ8XnrAs+/EjSiBEjNHDgQLVt21bt2rXT1KlTdfLkSd13333+Lg0AAFQyF0X46du3r44ePapnnnlGLpdLrVq10vLly0vcBA0AAHBRhB9JGjZs2G9+zIU/JyQkROPGjSvxUSEqB65P5cb1qdy4PpVbeV2fi+IhhwAAAKXFD5sCAABLIfwAAABLIfwAAABLIfwAAABLIfxAkjR9+nQ1aNBAVatWVfv27bVhw4bf7Pv222+rY8eOqlmzpmrWrKn4+Pjf7Y8/73yuzy8tWLBANpvN8wBQlI/zvT65ublKTk5W3bp1FRISoiZNmujTTz+toGqt53yvz9SpU3XFFVcoNDRUsbGxGj58uE6dOlVB1VrL6tWr1aNHD8XExMhms+nDDz/8w2O++OILtW7dWiEhIbr88ss1d+7c8x/YwPIWLFhggoODzT/+8Q+zY8cO8+CDD5rw8HCTnZ19zv733HOPmT59utmyZYvZtWuXGTRokHE4HOb777+v4Mqt4Xyvz1kHDx40l1xyienYsaPp2bNnxRRrQed7ffLz803btm1Nt27dzFdffWUOHjxovvjiC7N169YKrtwazvf6pKSkmJCQEJOSkmIOHjxoVqxYYerWrWuGDx9ewZVbw6effmqefPJJs2jRIiPJLF68+Hf7HzhwwFSrVs2MGDHC7Ny507zxxhsmMDDQLF++/LzGJfzAtGvXziQnJ3teFxUVmZiYGDNx4sRSHV9YWGjCwsLMvHnzyqtESyvL9SksLDTXXnuteeedd8zAgQMJP+XofK/PzJkzzWWXXWZOnz5dUSVa2vlen+TkZHPzzTd7tY0YMcJcd9115VonTKnCzxNPPGGuvPJKr7a+ffuahISE8xqLj70s7vTp08rIyFB8fLynLSAgQPHx8UpPTy/VOX7++WcVFBQoIiKivMq0rLJen/HjxysyMlKDBw+uiDItqyzX5+OPP5bT6VRycrKioqLUvHlzTZgwQUVFRRVVtmWU5fpce+21ysjI8Hw0duDAAX366afq1q1bhdSM35eenu51PSUpISGh1O9XZ100T3hG2fz4448qKioq8VMgUVFR+vbbb0t1jtGjRysmJqbE/yHx55Xl+nz11VeaPXu2tm7dWgEVWltZrs+BAwe0cuVKJSUl6dNPP9W+ffv08MMPq6CgQOPGjauIsi2jLNfnnnvu0Y8//qjrr79exhgVFhbqL3/5i/72t79VRMn4Ay6X65zX0+1263//+59CQ0NLdR5WfvCnTJo0SQsWLNDixYtVtWpVf5djecePH1f//v319ttvq3bt2v4uB+dQXFysyMhI/f3vf1ebNm3Ut29fPfnkk5o1a5a/S4PO3Ew7YcIEzZgxQ5s3b9aiRYu0dOlSPf/88/4uDT7Eyo/F1a5dW4GBgcrOzvZqz87OVnR09O8e+/LLL2vSpEn6/PPP1bJly/Is07LO9/rs379fhw4dUo8ePTxtxcXFkqSgoCDt3r1bjRo1Kt+iLaQs//zUrVtXVapUUWBgoKetWbNmcrlcOn36tIKDg8u1Zispy/V5+umn1b9/fz3wwAOSpBYtWujkyZMaMmSInnzySQUEsGbgT9HR0ee8nna7vdSrPhIrP5YXHBysNm3aKC0tzdNWXFystLQ0OZ3O3zxuypQpev7557V8+XK1bdu2Ikq1pPO9Pk2bNtX27du1detWz3bbbbepU6dO2rp1q2JjYyuy/IteWf75ue6667Rv3z5PKJWkPXv2qG7dugQfHyvL9fn5559LBJyzQdXwU5h+53Q6va6nJKWmpv7u+9U5nd+92LgYLViwwISEhJi5c+eanTt3miFDhpjw8HDjcrmMMcb079/fjBkzxtN/0qRJJjg42PzrX/8yWVlZnu348eP+msJF7Xyvz6/xba/ydb7XJzMz04SFhZlhw4aZ3bt3myVLlpjIyEjzwgsv+GsKF7XzvT7jxo0zYWFh5p///Kc5cOCA+eyzz0yjRo3MXXfd5a8pXNSOHz9utmzZYrZs2WIkmVdffdVs2bLFfPfdd8YYY8aMGWP69+/v6X/2q+6jRo0yu3btMtOnT+er7ii7N954w9SrV88EBwebdu3amXXr1nn23XjjjWbgwIGe1/Xr1zeSSmzjxo2r+MIt4nyuz68Rfsrf+V6ftWvXmvbt25uQkBBz2WWXmRdffNEUFhZWcNXWcT7Xp6CgwDz77LOmUaNGpmrVqiY2NtY8/PDD5qeffqr4wi1g1apV53w/OXtNBg4caG688cYSx7Rq1coEBwebyy67zMyZM+e8x7UZwzoeAACwDu75AQAAlkL4AQAAlkL4AQAAlkL4AQAAlkL4AQAAlkL4AQAAlkL4AQAAlkL4AVCujDEaMmSIIiIiZLPZtHXrVt1000167LHHfve4Bg0aaOrUqRVSIwBrIfwAFuVyufTII4/osssuU0hIiGJjY9WjR48Sv5vzZy1fvlxz587VkiVLlJWVpebNm2vRokUXza9kL168WB06dJDD4VBYWJiuvPLKPwx2APyLX3UHLOjQoUO67rrrFB4erpdeekktWrRQQUGBVqxYoeTkZH377bc+G2v//v2qW7eurr32Wk9bRESEz87vT2lpaerbt69efPFF3XbbbbLZbNq5c6dSU1PLbcyioiLZbDZ+XRz4M/7kz3IAuAB17drVXHLJJebEiRMl9v3yN4y+++47c9ttt5nq1aubsLAwc+edd3p+ENKYMz8CedVVV5l3333X1K9f39jtdtO3b1/jdruNMWd+l0e/+L2e+vXrG2PO/J7So48+6jlPdna26d69u6latapp0KCBee+990z9+vXNa6+95lXX4MGDTe3atU1YWJjp1KmT2bp1a6lrMcaYoqIiM3nyZNOoUSMTHBxsYmNjvX5QNDMz09x5553G4XCYmjVrmttuu80cPHjwN/+Ojz76qLnpppv+6M9tPv74Y9O2bVsTEhJiatWqZXr16uXZd+zYMdO/f38THh5uQkNDTZcuXcyePXs8++fMmWMcDof56KOPTLNmzUxgYKA5ePCgOXXqlBk5cqSJiYkx1apVM+3atTOrVq36w1oAGMN/OgAWc+zYMS1fvlzJycmqXr16if3h4eGSpOLiYvXs2VPHjh3Tl19+qdTUVB04cEB9+/b16r9//359+OGHWrJkiZYsWaIvv/xSkyZNkiS9/vrrGj9+vC699FJlZWVp48aN56xp0KBBOnz4sFatWqV//etfmjFjhnJycrz63HnnncrJydGyZcuUkZGh1q1bq3Pnzjp27FipapGksWPHatKkSXr66ae1c+dOzZ8/X1FRUZKkgoICJSQkKCwsTGvWrNHXX3+tGjVqqEuXLjp9+vQ5646OjtaOHTv0n//85zf/3kuXLtXtt9+ubt26acuWLUpLS1O7du285r5p0yZ9/PHHSk9PlzFG3bp1U0FBgafPzz//rMmTJ+udd97Rjh07FBkZqWHDhik9PV0LFizQN998ozvvvFNdunTR3r17f7MWAP8/f6cvABVr/fr1RpJZtGjR7/b77LPPTGBgoMnMzPS07dixw0gyGzZsMMacWW2pVq2a1+rKqFGjTPv27T2vX3vtNc+Kz1m/XPnZvXu31zmNMWbXrl1GkmflZ82aNcZut5tTp055nadRo0bmrbfeKlUtbrfbhISEmLfffvuc8/1//+//mSuuuMIUFxd72vLz801oaKhZsWLFOY85ceKE6datm2dVq2/fvmb27NledTqdTpOUlHTO4/fs2WMkma+//trT9uOPP5rQ0FDzwQcfGGPOrPxI8lrl+u6770xgYKD54YcfvM7XuXNnM3bs2HOOBeD/cM8PYDHGmFL127Vrl2JjYxUbG+tpi4uLU3h4uHbt2qVrrrlG0plvZYWFhXn61K1bt8SqzR+NExQUpDZt2njamjZt6lmBkqRt27bpxIkTqlWrltex//vf/7R//37P69+rZdeuXcrPz1fnzp3PWce2bdu0b98+r+Ml6dSpU15j/FL16tW1dOlS7d+/X6tWrdK6des0cuRIvf7660pPT1e1atW0detWPfjgg7879/bt23vaatWqpSuuuEK7du3ytAUHB6tly5ae19u3b1dRUZGaNGnidb78/PwSfyMAJRF+AItp3LixbDabz25qrlKlitdrm82m4uJin5z7rBMnTqhu3br64osvSuz7ZUj6vVpCQ0P/cIw2bdooJSWlxL46der87rGNGjVSo0aN9MADD+jJJ59UkyZN9P777+u+++77w3FLIzQ0VDabzavWwMBAZWRkKDAw0KtvjRo1/vR4wMWOe34Ai4mIiFBCQoKmT5+ukydPltifm5srSWrWrJkOHz6sw4cPe/bt3LlTubm5iouL81k9TZs2VWFhoTIyMjxtu3fv9tQhSa1bt5bL5VJQUJAuv/xyr6127dqlGqdx48YKDQ39za/yt27dWnv37lVkZGSJMRwOR6nn06BBA1WrVs3zt23ZsuVvjtmsWTMVFhZq/fr1nrb//ve/2r179+/+ja+++moVFRUpJyenRK3R0dGlrhWwKsIPYEHTp09XUVGR2rVrp3//+9/au3evdu3apWnTpsnpdEqS4uPj1aJFCyUlJWnz5s3asGGDBgwYoBtvvFFt27b1WS1XXHGFunTpooceekjr169XRkaGHnjgAa8Vk/j4eDmdTvXq1UufffaZDh06pLVr1+rJJ5/Upk2bSjVO1apVNXr0aD3xxBN69913tX//fq1bt06zZ8+WJCUlJal27drq2bOn1qxZo4MHD+qLL77QX//6V33//ffnPOezzz6rJ554Ql988YUOHjyoLVu26P7771dBQYFuueUWSdK4ceP0z3/+U+PGjdOuXbu0fft2TZ48WdKZQNazZ089+OCD+uqrr7Rt2zbde++9uuSSS9SzZ8/fnEuTJk2UlJSkAQMGaNGiRTp48KA2bNigiRMnaunSpaX6ewBWRvgBLOiyyy7T5s2b1alTJ40cOVLNmzfXLbfcorS0NM2cOVPSmY+MPvroI9WsWVM33HCD4uPjddlll+n999/3eT1z5sxRTEyMbrzxRvXu3VtDhgxRZGSkZ7/NZtOnn36qG264Qffdd5+aNGmifv366bvvvvN8W6s0nn76aY0cOVLPPPOMmjVrpr59+3ruCapWrZpWr16tevXqqXfv3mrWrJkGDx6sU6dOyW63n/N8N954ow4cOKABAwaoadOm6tq1q1wulz777DNdccUVkqSbbrpJCxcu1Mcff6xWrVrp5ptv1oYNG7zm3qZNG3Xv3l1Op1PGGH366aclPsI7199swIABGjlypK644gr16tVLGzduVL169Ur99wCsymZKe/cjAADARYCVHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCmEHwAAYCn/HyqBXbF2uSTjAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#qualitative study of those with high entropy\n",
        "df_high=data[data['entropy']>0.8]\n",
        "df_high['text'].tolist()[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GDXsD2fEhJvf",
        "outputId": "8b6dc3ff-d306-4c1e-e28b-40e1fedc8272"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Title: Sprint Set to Debut Video-Streaming Cell Phone Description: OVERLAND PARK, Kan. (AP) -- Channel surfing is moving off the couch as Sprint Corp...',\n",
              " 'Title: Monday Morning Description: The regular Monday Morning contributors will return to this page next week, as will the weekly poll.',\n",
              " 'Title: Playing the convergence game Description: Sony and Microsoft confront consumer apathy as they attempt to turn game consoles into multipurpose entertainment gadgets.',\n",
              " 'Title: Wherenet adds biz rules to RFID Description: The first indication that the value of RFID (radio frequency identification) will extend beyond the supply warehouse was shown last week by WhereNet, a provider of wireless location and communications technology.',\n",
              " 'Title: AdWatch: Bush Raps Kerry on Intel Reform (AP) Description: AP - Details of new television ad from President Bush to begin airing Monday:',\n",
              " \"Title: Jackson Squares Off With Attorney Description: SANTA MARIA, Calif. - Dozens of Michael Jackson's fans pressed against a chain-link fence outside court Monday, staking out viewing spots hours before the pop star was to arrive to face off against the prosecutor who has pursued him for years on child molestation charges...\",\n",
              " 'Title: Sybase upgrades PowerBuilder, plots RFID move Description: Sybase Inc. released a new version of its application development tool on Monday, called PowerBuilder 10, and outlined plans for an upgrade to its database software, Adaptive Server Enterprise, which will go into beta at the end of this month and is due for release next year.',\n",
              " 'Title: Phaithful Phlock to the Phinish as the 21-Year Jam Ends in Tears Description: Phish made its farewell in Vermont this past weekend in true Phish style: playing its own musical marathon, swarmed by adoring fans and isolated from the rest of the world.']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_low=data[data['entropy']<0.4]\n",
        "df_low.shape[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "saaQnjXcZirg",
        "outputId": "360381f9-27ec-4e62-d406-8f7570597469"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "321"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#length calculation for cost computation\n",
        "df_low=data[data['entropy']<0.2]\n",
        "print(df_low.shape[0])\n",
        "df=df_low\n",
        "df['length1']=df['prompt1'].apply(lambda x:len(x.split()))\n",
        "df['length2']=df['prompt2'].apply(lambda x:len(x.split()))\n",
        "df['length3']=df['prompt3'].apply(lambda x:len(x.split()))\n",
        "df['length4']=df['prompt4'].apply(lambda x:len(x.split()))\n",
        "df['length5']=df['prompt5'].apply(lambda x:len(x.split()))\n",
        "df['length6']=df['prompt6'].apply(lambda x:len(x.split()))\n",
        "df['length7']=df['prompt7'].apply(lambda x:len(x.split()))\n",
        "df['length8']=df['prompt8'].apply(lambda x:len(x.split()))\n",
        "df['length9']=df['prompt9'].apply(lambda x:len(x.split()))\n",
        "df['length10']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length11']=df['prompt11'].apply(lambda x:len(x.split()))\n",
        "df['length12']=df['prompt12'].apply(lambda x:len(x.split()))\n",
        "df['length13']=df['prompt13'].apply(lambda x:len(x.split()))\n",
        "df['length']=df['length1']+df['length2']+df['length3']+df['length4']+df['length5']+df['length6']+df['length7']+df['length8']+df['length9']+df['length10']+df['length11']+df['length12']+df['length13']\n",
        "#sum(df['length'].tolist())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kP2v4s8j79ru",
        "outputId": "574e4ce2-11ba-4aa2-f839-f843ef5b60f8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "266\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_low['text'].tolist()[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lljHhLxuaChh",
        "outputId": "efa3eca0-33ee-4f92-e36d-4aa983b54376"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\"Title: Wall St. Bears Claw Back Into the Black Description:  NEW YORK (Reuters) - Short-sellers, Wall Street's dwindling  band of ultra-cynics, are seeing green again.\",\n",
              " \"Title: Rescuing an Old Saver Description: If you think you may need to help your elderly relatives with their finances, don't be shy about having the money talk -- soon.\",\n",
              " 'Title: Veteran inventor in market float Description: Trevor Baylis, the veteran inventor famous for creating the Freeplay clockwork radio, is planning to float his company on the stock market.',\n",
              " 'Title: Quality Gets Swept Away Description: Quality Distribution is hammered after reporting a large loss for the second quarter.',\n",
              " \"Title: What's Cool In the Pool ... ... And Hot On the Deck Description: Americans are spending more on tricking out the places where they swim. Here's a look at the new wave of accessories\"]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_high=data[data['conf']>0.9]\n",
        "data_high.shape[0]/1154"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VvoR6jqffNPY",
        "outputId": "06820ec5-f21e-4d80-80e4-616ea331ef3b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5147313691507799"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data['gold_label']=data['Class Index']\n",
        "def calculate_alignment(threshold, df_input):\n",
        "  df_low=df_input[df_input['conf']>=threshold]\n",
        "  print(df_low.shape[0])\n",
        "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
        "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
        "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
        "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
        "  df_low_correct1=df_low[df_low['MV_world']==df_low['world_gold']]\n",
        "  df_low_correct2=df_low[df_low['MV_sports']==df_low['gold_sports']]\n",
        "  df_low_correct3=df_low[df_low['MV_business']==df_low['gold_business']]\n",
        "  df_low_correct4=df_low[df_low['MV_sci']==df_low['gold_sci']]\n",
        "  return (df_low_correct1.shape[0]+df_low_correct2.shape[0]+df_low_correct3.shape[0]+df_low_correct4.shape[0])/4/df_low.shape[0]"
      ],
      "metadata": {
        "id": "-wavQMyy9SoW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "calculate_alignment(0.6, data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qYvGm6fUBM5N",
        "outputId": "ab931b67-3806-49a8-9b49-1973877abed1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1064\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-33-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-33-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-33-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-33-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9224624060150376"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "x = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
        "y = [calculate_alignment(i, data) for i in x]\n",
        "\n",
        "plt.scatter(x, y, s=80, facecolors='none', edgecolors='r')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "6UY04Z4OV61g",
        "outputId": "9beb2ead-f02c-4462-eeef-3d1d6d5152a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1154\n",
            "1153\n",
            "1139\n",
            "1127\n",
            "1088\n",
            "1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-7-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-7-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-7-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-7-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-7-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-7-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-7-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-7-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-7-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-7-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-7-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-7-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-7-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-7-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-7-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-7-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-7-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-7-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-7-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-7-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApe0lEQVR4nO3df1yV9d3H8Tdgh0Pjh5aJgpRBNvsJKcqNrlyLjUZz1u2W3ZkSWz9MdCvue4ZJ4nSF9ShvCyl/rNKZTVupa2U0I6vbZtkUtkrtx/BWo0DdXRzC5Nf53n9cD48jQTkohy/H1/PxuB7M63yvw+f6jjhvvtf3+l4hxhgjAAAAi4V2dwEAAADHQ2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFivV3cXcLJ4vV599tlnioqKUkhISHeXAwAAOsAYo7q6OsXFxSk0tP1xlKAJLJ999pkSEhK6uwwAANAJe/fu1cCBA9t9PWgCS1RUlCTnhKOjo7u5GgAA0BEej0cJCQm+z/H2BE1gOXwZKDo6msACAEAPc7zpHEy6BQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwXtDc1gwAAE4yr1d6803p44+d/z1okHTVVVKvwMcHAgsAAGitpUVauNDZPvlECglxNq9XGjhQmjxZ+s//lNzugJXEJSEAAHBEY6M0bpyUlycNH+6MsDQ3S01N0tat0tVXS3PnSt//vlRXF7Cy/A4sb775psaMGaO4uDiFhIRo3bp1xz3m9ddf19ChQxUeHq7zzjtPy5YtO6pNSUmJBg0aJLfbrbS0NG3ZssXf0gAAwIn6xS+kl1+WXnhBeuYZ6fLLpdBQZxs6VFq6VNq4Ufr736UbbwxYWX4Hlvr6eiUnJ6ukpKRD7Xft2qVrrrlGV155pSoqKnTnnXfqlltu0SuvvOJrs3r1auXl5amwsFDbtm1TcnKyMjMztW/fPn/LAwAAnbV7txNIHnxQuuaa9tulp0u//a304ovSu+8GpLQQY4zp9MEhIVq7dq2uvfbadtvcfffdeumll/T+++/79t1www368ssvVVpaKklKS0vT8OHDtXDhQkmS1+tVQkKCpk2bpvz8/A7V4vF4FBMTo9raWp4lBABAZ8yc6cxbqaqSIiOP3balRUpKkr77XamNKycd1dHP7y6fw7J582ZlZGS02peZmanNmzdLkhobG7V169ZWbUJDQ5WRkeFr05aGhgZ5PJ5WGwAAOAEbNkhjxx4/rEhSWJh0ww3Sq692fV0KQGCprq5WbGxsq32xsbHyeDz6+uuvdeDAAbW0tLTZprq6ut33LSoqUkxMjG9LSEjokvoBADhl1NVJfft2vH3fvlKABgx67F1CM2bMUG1trW/bu3dvd5cEAEDPFhMj1dR0vP2+fc4xAdDl67D0799fNd84+ZqaGkVHRysiIkJhYWEKCwtrs03//v3bfd/w8HCFh4d3Sc0AAJySrr5aeughZ9TkePNBm5udu4iysgJSWpePsKSnp6usrKzVvg0bNig9PV2S5HK5NGzYsFZtvF6vysrKfG0AAEAA3HqrdOiQtGjR8duuWuVMzp0ypevrUicCy1dffaWKigpVVFRIcm5brqio0J49eyQ5l2omTZrkaz958mRVVlZq+vTp2rlzpx577DE9++yzuuuuu3xt8vLytHTpUi1fvlw7duzQHXfcofr6euXk5Jzg6QEAgA6Lj5dyc6V77pGee679dhs3SrffLv3kJ1JKSmBqM37auHGjkXTUlp2dbYwxJjs724wePfqoY1JSUozL5TKJiYnmqaeeOup9i4uLzdlnn21cLpcZMWKEefvtt/2qq7a21kgytbW1/p4SAAA4rKnJmBtvNCYkxJgxY4x5+WVj6uqM+eorY954w5gbbjCmVy9jvv99Y+rrT/jbdfTz+4TWYbEJ67AAAHCSeL3O2iqPPir97W+tXxs82BmFmTJFOu20E/5WHf38JrAAAIC2GeM8P+jw05rPOUcaOdJZpv8k6ejnN09rBgAAbQsJkVJTna2b9dh1WAAAwKmDwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1utUYCkpKdGgQYPkdruVlpamLVu2tNu2qalJc+bMUVJSktxut5KTk1VaWtqqTUtLi+69916de+65ioiIUFJSkubOnStjTGfKAwAAQcbvwLJ69Wrl5eWpsLBQ27ZtU3JysjIzM7Vv37422xcUFGjx4sUqLi7W9u3bNXnyZF133XUqLy/3tXnggQf0+OOPa+HChdqxY4ceeOABPfjggyouLu78mQEAgKARYvwcxkhLS9Pw4cO1cOFCSZLX61VCQoKmTZum/Pz8o9rHxcVp5syZys3N9e0bN26cIiIi9PTTT0uSfvSjHyk2NlZPPPFEu22Ox+PxKCYmRrW1tYqOjvbnlAAAQDfp6Oe3XyMsjY2N2rp1qzIyMo68QWioMjIytHnz5jaPaWhokNvtbrUvIiJCmzZt8v175MiRKisr00cffSRJ+tvf/qZNmzbphz/8Ybu1NDQ0yOPxtNoAAEBw6uVP4wMHDqilpUWxsbGt9sfGxmrnzp1tHpOZman58+friiuuUFJSksrKyrRmzRq1tLT42uTn58vj8WjIkCEKCwtTS0uL7rvvPk2YMKHdWoqKivTrX//an/IBAEAP1eV3CT3yyCMaPHiwhgwZIpfLpalTpyonJ0ehoUe+9bPPPquVK1fqmWee0bZt27R8+XI99NBDWr58ebvvO2PGDNXW1vq2vXv3dvWpAACAbuLXCEvfvn0VFhammpqaVvtramrUv3//No8566yztG7dOh06dEj//Oc/FRcXp/z8fCUmJvra/OpXv1J+fr5uuOEGSdIll1yi3bt3q6ioSNnZ2W2+b3h4uMLDw/0pHwAA9FB+jbC4XC4NGzZMZWVlvn1er1dlZWVKT08/5rFut1vx8fFqbm7W888/r7Fjx/peO3jwYKsRF0kKCwuT1+v1pzwAABCk/BphkaS8vDxlZ2crNTVVI0aM0IIFC1RfX6+cnBxJ0qRJkxQfH6+ioiJJ0jvvvKOqqiqlpKSoqqpKs2fPltfr1fTp033vOWbMGN133306++yzddFFF6m8vFzz58/Xz372s5N0mgAAoCfzO7CMHz9e+/fv16xZs1RdXa2UlBSVlpb6JuLu2bOn1WjJoUOHVFBQoMrKSkVGRiorK0srVqxQ7969fW2Ki4t17733asqUKdq3b5/i4uJ0++23a9asWSd+hgAAoMfzex0WW7EOCwAAPU+XrMMCAADQHQgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALBepwJLSUmJBg0aJLfbrbS0NG3ZsqXdtk1NTZozZ46SkpLkdruVnJys0tLSo9pVVVXppptu0plnnqmIiAhdcskl+utf/9qZ8gAAQJDxO7CsXr1aeXl5Kiws1LZt25ScnKzMzEzt27evzfYFBQVavHixiouLtX37dk2ePFnXXXedysvLfW2++OILjRo1Sqeddppefvllbd++XQ8//LD69OnT+TMDAABBI8QYY/w5IC0tTcOHD9fChQslSV6vVwkJCZo2bZry8/OPah8XF6eZM2cqNzfXt2/cuHGKiIjQ008/LUnKz8/XW2+9pf/5n//p9Il4PB7FxMSotrZW0dHRnX4fAAAQOB39/PZrhKWxsVFbt25VRkbGkTcIDVVGRoY2b97c5jENDQ1yu92t9kVERGjTpk2+f7/wwgtKTU3VT3/6U/Xr10+XXXaZli5desxaGhoa5PF4Wm0AACA4+RVYDhw4oJaWFsXGxrbaHxsbq+rq6jaPyczM1Pz58/Xxxx/L6/Vqw4YNWrNmjT7//HNfm8rKSj3++OMaPHiwXnnlFd1xxx36xS9+oeXLl7dbS1FRkWJiYnxbQkKCP6cCAAB6kC6/S+iRRx7R4MGDNWTIELlcLk2dOlU5OTkKDT3yrb1er4YOHar7779fl112mW677TbdeuutWrRoUbvvO2PGDNXW1vq2vXv3dvWpAACAbuJXYOnbt6/CwsJUU1PTan9NTY369+/f5jFnnXWW1q1bp/r6eu3evVs7d+5UZGSkEhMTfW0GDBigCy+8sNVxF1xwgfbs2dNuLeHh4YqOjm61AQCA4ORXYHG5XBo2bJjKysp8+7xer8rKypSenn7MY91ut+Lj49Xc3Kznn39eY8eO9b02atQoffjhh63af/TRRzrnnHP8KQ8AAASpXv4ekJeXp+zsbKWmpmrEiBFasGCB6uvrlZOTI0maNGmS4uPjVVRUJEl65513VFVVpZSUFFVVVWn27Nnyer2aPn267z3vuusujRw5Uvfff7+uv/56bdmyRUuWLNGSJUtO0mkCAICezO/AMn78eO3fv1+zZs1SdXW1UlJSVFpa6puIu2fPnlbzUw4dOqSCggJVVlYqMjJSWVlZWrFihXr37u1rM3z4cK1du1YzZszQnDlzdO6552rBggWaMGHCiZ8hAADo8fxeh8VWrMMCAEDP0yXrsAAAAHQHAgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArNepwFJSUqJBgwbJ7XYrLS1NW7ZsabdtU1OT5syZo6SkJLndbiUnJ6u0tLTd9vPmzVNISIjuvPPOzpQGAACCkN+BZfXq1crLy1NhYaG2bdum5ORkZWZmat++fW22Lygo0OLFi1VcXKzt27dr8uTJuu6661ReXn5U23fffVeLFy/WpZde6v+ZAACAoOV3YJk/f75uvfVW5eTk6MILL9SiRYt0+umn68knn2yz/YoVK3TPPfcoKytLiYmJuuOOO5SVlaWHH364VbuvvvpKEyZM0NKlS9WnT5/OnQ0AAAhKfgWWxsZGbd26VRkZGUfeIDRUGRkZ2rx5c5vHNDQ0yO12t9oXERGhTZs2tdqXm5ura665ptV7H0tDQ4M8Hk+rDQAABCe/AsuBAwfU0tKi2NjYVvtjY2NVXV3d5jGZmZmaP3++Pv74Y3m9Xm3YsEFr1qzR559/7muzatUqbdu2TUVFRR2upaioSDExMb4tISHBn1MBAAA9SJffJfTII49o8ODBGjJkiFwul6ZOnaqcnByFhjrfeu/evfrlL3+plStXHjUScywzZsxQbW2tb9u7d29XnQIAAOhmfgWWvn37KiwsTDU1Na3219TUqH///m0ec9ZZZ2ndunWqr6/X7t27tXPnTkVGRioxMVGStHXrVu3bt09Dhw5Vr1691KtXL73xxht69NFH1atXL7W0tLT5vuHh4YqOjm61AQCA4ORXYHG5XBo2bJjKysp8+7xer8rKypSenn7MY91ut+Lj49Xc3Kznn39eY8eOlSRdddVVeu+991RRUeHbUlNTNWHCBFVUVCgsLKwTpwUAAIJJL38PyMvLU3Z2tlJTUzVixAgtWLBA9fX1ysnJkSRNmjRJ8fHxvvko77zzjqqqqpSSkqKqqirNnj1bXq9X06dPlyRFRUXp4osvbvU9vvWtb+nMM888aj8AADg1+R1Yxo8fr/3792vWrFmqrq5WSkqKSktLfRNx9+zZ45ufIkmHDh1SQUGBKisrFRkZqaysLK1YsUK9e/c+aScBAACCW4gxxnR3ESeDx+NRTEyMamtrmc8CAEAP0dHPb54lBAAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6nQosJSUlGjRokNxut9LS0rRly5Z22zY1NWnOnDlKSkqS2+1WcnKySktLW7UpKirS8OHDFRUVpX79+unaa6/Vhx9+2JnSAABAEPI7sKxevVp5eXkqLCzUtm3blJycrMzMTO3bt6/N9gUFBVq8eLGKi4u1fft2TZ48Wdddd53Ky8t9bd544w3l5ubq7bff1oYNG9TU1KQf/OAHqq+v7/yZAQCAoBFijDH+HJCWlqbhw4dr4cKFkiSv16uEhARNmzZN+fn5R7WPi4vTzJkzlZub69s3btw4RURE6Omnn27ze+zfv1/9+vXTG2+8oSuuuKJDdXk8HsXExKi2tlbR0dH+nBIAAOgmHf389muEpbGxUVu3blVGRsaRNwgNVUZGhjZv3tzmMQ0NDXK73a32RUREaNOmTe1+n9raWknSGWec0W6bhoYGeTyeVhsAAAhOfgWWAwcOqKWlRbGxsa32x8bGqrq6us1jMjMzNX/+fH388cfyer3asGGD1qxZo88//7zN9l6vV3feeadGjRqliy++uN1aioqKFBMT49sSEhL8ORUAANCDdPldQo888ogGDx6sIUOGyOVyaerUqcrJyVFoaNvfOjc3V++//75WrVp1zPedMWOGamtrfdvevXu7onwAAGABvwJL3759FRYWppqamlb7a2pq1L9//zaPOeuss7Ru3TrV19dr9+7d2rlzpyIjI5WYmHhU26lTp+rFF1/Uxo0bNXDgwGPWEh4erujo6FYbAAAITn4FFpfLpWHDhqmsrMy3z+v1qqysTOnp6cc81u12Kz4+Xs3NzXr++ec1duxY32vGGE2dOlVr167Va6+9pnPPPdfP0wAAAMGsl78H5OXlKTs7W6mpqRoxYoQWLFig+vp65eTkSJImTZqk+Ph4FRUVSZLeeecdVVVVKSUlRVVVVZo9e7a8Xq+mT5/ue8/c3Fw988wz+uMf/6ioqCjffJiYmBhFREScjPMEAAA9mN+BZfz48dq/f79mzZql6upqpaSkqLS01DcRd8+ePa3mpxw6dEgFBQWqrKxUZGSksrKytGLFCvXu3dvX5vHHH5ckffe73231vZ566indfPPN/p8VAAAIKn6vw2Ir1mEBAKDn6ZJ1WAAAALoDgQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1enV3AQAA+O3rr6X166VPP5V69ZIuvFAaPVoK5e/wYEVgAQD0HF9+Kf3mN9KTT0pffCFFREjNzVJTk3T++dIvfiHdcQfBJQjx/ygAoGeoqZG+8x1pyRLpZz+TPv5YOnhQamiQ3nxTuuwyado0aeJEqaWlu6vFScYICwDAfi0t0o9/LP3f/0lbtkhDhhx5LSREuvxyZxs3TrrhBmngQOmBB7qvXpx0jLAAAOy3fr0TVJ59tnVY+aaf/lS6917p0Uelf/4zcPWhyxFYAAD2e+wxafhw55LQ8UyZInm90lNPdX1dCBguCQGnir/9TfrznyWPR4qKkjIzpeTk7q4qOFVVSQcOOBNCBw2SXK7urqhnM0YqK5PmzetY+379pIwM6bXXpP/6r66tDQHDCAsQ7MrKnL9KU1KkX/9aWr5cmjvX+feoUdKGDd1dYXBobJSeflpKT3fmT6SkSN/+tpSQIBUUOLffonOampztjDM6fsyZZ0p1dV1XEwKOwAIEs9/9TvrBD5y/UJ97zrkNdM8eZ+Li8887kxWvvlpatqy7K+3ZvvjC+Yt+4kRn9GrVKuntt6VXX5Wuv96ZT3HRRU54hP9OO01yu527hDqqulqKiem6mhBwIcYY091FnAwej0cxMTGqra1VdHR0d5cDdL+//EW64grp5pulxYulsLCj27S0OGtWPPmktHGjc5cF/NPYKF11lbR9u/TCC86o1TfV1jrBZdMm6Y03pNTUwNfZ040bJ+3cKb3/vhO0j2XvXudS3MKFzs83rNbRz28CCxCsxo6Vdu2Stm1zVgJtT0uLNGyYcxnjxRcDV1+wWLZMysmR3npLGjmy/XZff+283ru3Ew7hn9dec4Lhn/4k/ehHx257113Sb38rffaZM+IFq3X085tLQkAw2r3bCR9Tpx47rEjOyMvUqc5to7t2Baa+YPLYY85ltWOFFcmZgJufL73+uvTBBwEpLahceaVzefOmm6TNm9tvt3ChtGCBM2+IsBJUCCxAMNq82bmt8yc/6Vj7n/7Umefyl790bV3BZvdu6d13pVtu6Vj7666T+vRx5hPBPyEh0h/+IF1yifPMoBtvdFa3/fJLaf9+afVqZ/+0aVJenjR9endXjJOM25rRvTweaeVKZ4LiwYNS377OtervfY9ngZyI+nrna0cnHR4ehj18HDrmwAHna2Jix9q7XM5dQ/v3d11NwSw62rmr7fHHnZGt3/++9eujR0tr1jjBEEGHwNIeY5wP0Q8+cB6sFR/vDEeGh3d3ZcGhudlZjXLhQufa/ogRUmSks1bIokXS4MHOsG5WVndX2jP16eN8/fRT6Zxzjt++qsr56s9to3DuXJH8C3r19c7lIXSO2+3MUfnlL53f0VVVzmXPCy449gq46PEILN9kjLM64oIF0nvvOfvCwpyJiWed5Qz95ucf+YsU/mtpkcaPl/74R+nuu51Z/AMHOq8Z40xenDtXGjPGWdfiP/6je+vtia66SvrWt5wJoYWFx2//1FPS6ac7t+ai45KSnHD4xz92bAXW99+X/vEPZ8VWnJjQ0OPPG0JQYcz9X3m90q23Sj//uTPE++c/H1mw6IMPnA/ORx91bv3ct6+7q+257r/f+QW/dq10331HworkXKf+znecCaA33SRlZzu3i8I/MTFO/y1adOSyRXv++U9niH3CBOcOFnSc2+08NfjJJzs2yrJwodS/v3TttV1eGhBsCCz/6te/dn7xLF8urVsnff/7zlBjSIh04YXSI49I77zjLF40diyPL++MhgapuNh51seYMe23CwuTli515rSUlASuvmCSn+/8jGZltT9n4sAB6ZprnFA+Y0Zg6wsWubnSoUPOJNCGhvbb/f730pIlzuUMluoH/EZgOezLL6WHHnJ+yU+a1H67iy5ynhb69tvSyy8HrLygsXat8+GZm3v8ti6XM+L1u99JX33V9bUFm0GDpNJS506WwYOda/7vvuv8+69/le68UzrvPKmy0vlZPvfc7q64Zzr3XOfuldJSZ6G+deucOVqH7djh/LxPmOCshMuzbYBOIbAc9rvfOStWTpt2/LZXXOGsVMlf/v4rL3eu+3/72x1rn5XlhJV//KNr6wpWQ4c6fX7HHdIzzziTmwcNcuZQrFwpTZ7svM7KqycmK8tZ2Cw01LlDJT7eWYxvyBBndPa556SiImeuEHe/AZ3CpNvDNmxwbqUdMKBj7W+80bnP35jjLxONIxoa/LvT6vBdGMcaasexxcU5H5azZzvhxONxJo2npBzpX5y4UaOc9W+2bXOe03T4ac2zZ0v//u9cBgJOEIHlsLo656+ijurb1xn2PXSIWxT90b+/85yPgwedu1KO58MPna+xsV1b16kgPFz6t3/r7iqC39ChzgbgpGJs8rCYGP+eBLpv35EniKLjrr/eucSzenXH2i9Z4ty62JG1RAAAQYvActjVVzsPJPv00+O3NcaZ8/LDH3I5yF+JiU5fz5vnPMH2WMrKnI2nrQLAKY/ActhNNzmXKP77v4/f9tVXpb//3bk1F/57+GFnhCoz03maalteftmZvPiDH0g33BDY+gAA1mEOy2FRUdI99zjb4MHO3RNt2brVWUBu9GhnnRb474ILnEnOP/qRc8fKT37ihJPISOeW2yeecG67vfpq53bR4z1tGAAQ9Pgk+Ff5+VJ1tXMJ4rnnnLUTrrrKmavy/vvOqqErV0rJyc4Dtrg9sfNSU531KZYtc1ZZPfwQs5AQ51Lbn/7kfA0L69YyAQB2CDHGmO4u4mTweDyKiYlRbW2tok/kOT/GOH/VL1jg3KL4rxISnJGXO+/s2B0u6BhjnOXhv/7aeS5LZGR3VwQACJCOfn4TWI7l7393nmPT1OTc8jx6NH/xAwBwEnX085tLQsdy6aXOBgAAuhWTMAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1guadVgOr3/n8Xi6uRIAANBRhz+3j7eObdAElrq6OklSQkJCN1cCAAD8VVdXp5iYmHZfD5ql+b1erz777DNFRUUpJCTkpL2vx+NRQkKC9u7de/KW/MdR6OfAoa8Dg34ODPo5MLqyn40xqqurU1xcnEKP8VDhoBlhCQ0N1cCBA7vs/aOjo/mPIQDo58ChrwODfg4M+jkwuqqfjzWychiTbgEAgPUILAAAwHoEluMIDw9XYWGhwsPDu7uUoEY/Bw59HRj0c2DQz4FhQz8HzaRbAAAQvBhhAQAA1iOwAAAA6xFYAACA9QgsAADAegQWSSUlJRo0aJDcbrfS0tK0ZcuWY7b/wx/+oCFDhsjtduuSSy7R+vXrA1Rpz+ZPPy9dulSXX365+vTpoz59+igjI+O4/7/A4e/P82GrVq1SSEiIrr322q4tMIj429dffvmlcnNzNWDAAIWHh+v888/n90cH+NvPCxYs0Le//W1FREQoISFBd911lw4dOhSganumN998U2PGjFFcXJxCQkK0bt264x7z+uuva+jQoQoPD9d5552nZcuWdW2R5hS3atUq43K5zJNPPmk++OADc+utt5revXubmpqaNtu/9dZbJiwszDz44INm+/btpqCgwJx22mnmvffeC3DlPYu//XzjjTeakpISU15ebnbs2GFuvvlmExMTYz799NMAV96z+NvPh+3atcvEx8ebyy+/3IwdOzYwxfZw/vZ1Q0ODSU1NNVlZWWbTpk1m165d5vXXXzcVFRUBrrxn8befV65cacLDw83KlSvNrl27zCuvvGIGDBhg7rrrrgBX3rOsX7/ezJw506xZs8ZIMmvXrj1m+8rKSnP66aebvLw8s337dlNcXGzCwsJMaWlpl9V4ygeWESNGmNzcXN+/W1paTFxcnCkqKmqz/fXXX2+uueaaVvvS0tLM7bff3qV19nT+9vM3NTc3m6ioKLN8+fKuKjEodKafm5ubzciRI81vf/tbk52dTWDpIH/7+vHHHzeJiYmmsbExUCUGBX/7OTc313zve99rtS8vL8+MGjWqS+sMJh0JLNOnTzcXXXRRq33jx483mZmZXVbXKX1JqLGxUVu3blVGRoZvX2hoqDIyMrR58+Y2j9m8eXOr9pKUmZnZbnt0rp+/6eDBg2pqatIZZ5zRVWX2eJ3t5zlz5qhfv376+c9/Hogyg0Jn+vqFF15Qenq6cnNzFRsbq4svvlj333+/WlpaAlV2j9OZfh45cqS2bt3qu2xUWVmp9evXKysrKyA1nyq647MwaB5+2BkHDhxQS0uLYmNjW+2PjY3Vzp072zymurq6zfbV1dVdVmdP15l+/qa7775bcXFxR/0HgiM608+bNm3SE088oYqKigBUGDw609eVlZV67bXXNGHCBK1fv16ffPKJpkyZoqamJhUWFgai7B6nM/1844036sCBA/rOd74jY4yam5s1efJk3XPPPYEo+ZTR3mehx+PR119/rYiIiJP+PU/pERb0DPPmzdOqVau0du1aud3u7i4naNTV1WnixIlaunSp+vbt293lBD2v16t+/fppyZIlGjZsmMaPH6+ZM2dq0aJF3V1aUHn99dd1//3367HHHtO2bdu0Zs0avfTSS5o7d253l4YTdEqPsPTt21dhYWGqqalptb+mpkb9+/dv85j+/fv71R6d6+fDHnroIc2bN0+vvvqqLr300q4ss8fzt5//8Y9/6H//9381ZswY3z6v1ytJ6tWrlz788EMlJSV1bdE9VGd+pgcMGKDTTjtNYWFhvn0XXHCBqqur1djYKJfL1aU190Sd6ed7771XEydO1C233CJJuuSSS1RfX6/bbrtNM2fOVGgof6efDO19FkZHR3fJ6Ip0io+wuFwuDRs2TGVlZb59Xq9XZWVlSk9Pb/OY9PT0Vu0lacOGDe22R+f6WZIefPBBzZ07V6WlpUpNTQ1EqT2av/08ZMgQvffee6qoqPBtP/7xj3XllVeqoqJCCQkJgSy/R+nMz/SoUaP0ySef+EKhJH300UcaMGAAYaUdnenngwcPHhVKDodEw6PzTppu+Szssum8PcSqVatMeHi4WbZsmdm+fbu57bbbTO/evU11dbUxxpiJEyea/Px8X/u33nrL9OrVyzz00ENmx44dprCwkNuaO8Dffp43b55xuVzmueeeM59//rlvq6ur665T6BH87edv4i6hjvO3r/fs2WOioqLM1KlTzYcffmhefPFF069fP/Ob3/ymu06hR/C3nwsLC01UVJT5/e9/byorK82f//xnk5SUZK6//vruOoUeoa6uzpSXl5vy8nIjycyfP9+Ul5eb3bt3G2OMyc/PNxMnTvS1P3xb869+9SuzY8cOU1JSwm3NgVBcXGzOPvts43K5zIgRI8zbb7/te2306NEmOzu7Vftnn33WnH/++cblcpmLLrrIvPTSSwGuuGfyp5/POeccI+morbCwMPCF9zD+/jz/KwKLf/zt67/85S8mLS3NhIeHm8TERHPfffeZ5ubmAFfd8/jTz01NTWb27NkmKSnJuN1uk5CQYKZMmWK++OKLwBfeg2zcuLHN37mH+zY7O9uMHj36qGNSUlKMy+UyiYmJ5qmnnurSGkOMYYwMAADY7ZSewwIAAHoGAgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArPf/bqpJKOlO1PwAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sm1_nA_tWAry",
        "outputId": "4bd91773-002a-458b-999a-9f71a7e081ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.9057625649913345,\n",
              " 0.9061144839549002,\n",
              " 0.9080333625987709,\n",
              " 0.907941437444543,\n",
              " 0.9096966911764706,\n",
              " 1.0]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "x = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
        "y = [0.9057625649913345, 0.9061144839549002, 0.9080333625987709, 0.907941437444543, 0.9096966911764706, 1.0]\n",
        "\n",
        "sns.set_style(\"darkgrid\")  # Set the plot style to 'darkgrid' with gray background\n",
        "sns.scatterplot(x=x, y=y, color='orange')\n",
        "plt.xlabel('Self-Confidence Threshold')\n",
        "plt.ylabel('Alignment with human label')\n",
        "plt.title('AG News')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "M32PAsFVWZ3J",
        "outputId": "34c6364f-ed0a-4ade-d4bd-597ad72c9903"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGMElEQVR4nO3deVxVdf7H8fcFWcZdwKUMtwxEATfQNIpG0dLCSR3SykyzXNJfZYtb2qRplKOTueWWu7mkaTqSo+WUVoj7ruWW4uQ4LOaCynp+f/Tg2g0yLt1zkO7r+Xj4eMA53/O9n/Mh4+1ZbYZhGAIAAHAjHiVdAAAAgNUIQAAAwO0QgAAAgNshAAEAALdDAAIAAG6HAAQAANwOAQgAALgdAhAAAHA7BCAAAOB2CEAAAMDtEIAAWGrJkiUKDg5WXFzcTcelpaVpwoQJio2NVdOmTRUWFqZ27dpp+PDh2rlz529+ztmzZxUcHKzg4GD961//KrB+ypQpCg4OVnp6erH3BUDpVaakCwDgXtatW6eaNWtq//79On36tGrXrl1gzP79+9W3b19lZGTooYceUvfu3eXt7a2zZ8/qs88+08cff6zFixcrMjKySJ85bdo0tW/fXjabzdW7A6CUIgABsExycrL27NmjqVOn6vXXX9e6des0aNAghzEXL17Uc889pzJlymjNmjW68847Hda/+OKLWr9+vXx8fIr0mSEhITpy5Ig2bdqk9u3bu2xfAJRunAIDYJl169apUqVKio6O1gMPPKB169YVGLNs2TKlpKRoxIgRBcKPJNlsNj388MMKDw8v0md27NhRderU0bRp02QYxm+O37dvn/r06aPmzZurcePG6tGjh3bt2mVff/ToUQUHB+vzzz+3Lzt48KCCg4PVuXNnh7meeeYZh1N9Bw4cUJ8+fdSyZUuFh4erTZs2Gj58eJH2A4BrEYAAWGbdunVq166dvL299fDDD+v777/X/v37Hcb8+9//lq+vr9q1a+eSz/T09NSAAQN09OhRbdq06aZjExMT9cQTTygjI0ODBg3S4MGDdenSJT311FP2OoOCglSxYkWH65B27twpDw8PHT16VFeuXJEk5eXlac+ePYqIiJD00zVNffr00dmzZ9W3b1+NGjVKsbGx2rdvn0v2E4BzCEAALHHw4EGdPHlSDz30kCSpefPmqlGjRoGjQCdPnlTdunXl5eXlsPzKlStKT0+3/7l69WqRPzs2NvY3jwIZhqE33nhDLVu21NKlS9W7d2/16tVLK1asUPXq1TVp0iRJkoeHh5o1a+YQgHbt2qWYmBjZbDbt3r1bkuxhKD8A7dmzRxcvXtSECRPUp08fxcXFafDgwUpISCjyfgBwHQIQAEusW7dOAQEBatmypaSfTmV17NhRCQkJys3NtY+7cuWKypYtW2D7IUOGqFWrVvY/EyZMKPJn//wo0GeffVbomCNHjuj7779XbGysLly44BC0WrVqpR07digvL0/ST+Ht8OHD9hC2a9cu3XfffWrQoIH9dNnOnTtls9nUvHlzSVKFChUkSV988YWys7OLXDsAc3ARNADT5ebmav369WrZsqXOnj1rXx4eHq65c+cqMTFRUVFRkqRy5coVenTn+eefV48ePSRJvXv3drqG2NhYTZ8+XdOmTVNMTEyB9d9//70kaejQob86x+XLl1WpUiVFREQoJydHe/fuVY0aNZSWlqaIiAgdP37cfmRo586dql+/vipXrixJatGihR544AFNnTpV8+fPV4sWLRQTE6PY2Fh5e3s7vT8Afh8CEADTbdu2TSkpKVq/fr3Wr19fYP26devsAahevXo6evSosrOzHU6DNWjQ4HfVkH8UaNiwYQ4XMOfLPzU2ZMgQhYSEFDpH/pGp0NBQ+fj4aMeOHbr99tvl7++vunXrKiIiQh9++KGysrLsp8Xy2Ww2TZ48WXv37tW///1vbd26VSNGjNC8efO0fPlylStX7nftHwDnEIAAmG7dunXy9/fX66+/XmDdpk2btGnTJo0ePVq+vr66//77tXfvXm3atEkdO3Z0aR2dOnXS+++/r6lTp6pNmzYO6wIDAyVJ5cuXV+vWrW86j7e3t8LDw7Vz507dfvvt9ut8mjdvrqysLK1du1apqamFPqeoSZMmatKkiQYPHqx169bplVdeUUJCwm8+GBKAa3ENEABTXb9+XRs3btT999+vBx98sMCf/LuuNm/eLEl67LHHFBAQoPj4eJ06darAfEW5lf3X5B8FOnLkiP3z8oWGhqpWrVqaO3euMjIyCmz7yydGN2/eXPv371dSUpL9Oh8/Pz/deeedmj17tiTZg5H00/ONfll7/pGmrKysYu8TgOLhCBAAU23evFkZGRkFjrjka9Kkifz8/LR27Vp17NhRlStX1tSpU9W/f3/95S9/0UMPPaTQ0FB5eXnp3Llz2rBhgyTptttuK1Y9+dcCHTlyxGG5h4eHxo4dq2effVYPP/ywunTpourVq+v8+fNKSkpS+fLlNWPGDPv4iIgIzZgxQ+fOnXMIOhEREVq+fLlq1qypGjVq2JevXr1aS5cuVUxMjGrVqqWMjAytWLFC5cuX13333VesfQFQfAQgAKZau3atfHx8dM899xS63sPDQ/fff7/WrVunCxcuqEqVKmratKn++c9/at68efryyy+VkJCgvLw8Va9eXc2bN9ebb77pEDqcUaZMGQ0YMKDQBxC2bNlSy5cv1/Tp07V48WJdvXpVVatWVXh4uLp16+YwtmnTpvL09JSvr6/D9Un5AeiX9bVo0UIHDhxQQkKCUlNTVaFCBYWHh2vChAn2028ArGMzfs/xZAAAgFKIa4AAAIDbIQABAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwOwQgAADgdghAAADA7fAk6JtIS7ssVz8m0maT/P0rmDI3bqDP1qDP1qDP1qDP1jGr1/nzFgUB6CYMQ6b9JTBzbtxAn61Bn61Bn61Bn61Tkr3mFBgAAHA7BCAAAOB2CEAAAMDtEIAAAIDbIQABAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwOwQgAADgdko0AO3YsUP9+/dXVFSUgoOD9dlnn/3mNklJSercubNCQ0PVrl07ffzxxwXGLFmyRG3atFFYWJji4uK0f/9+M8oHAABOKueVoSqeyVJqkqp4nlU5r4wSqaNEA9DVq1cVHBysv/3tb0Uan5ycrH79+qlly5b65JNP9NRTT2nkyJHaunWrfUxCQoLi4+M1cOBArV69Wg0aNFCfPn2UlpZm1m4AAIAiqOydpj/tfkqenzaSNt4tz08b6k+7n1Jlb+t/R5doAIqOjtbgwYPVrl27Io1ftmyZ7rjjDg0bNkx33nmnevTooQceeEDz58+3j5k3b54effRRde3aVfXr19fo0aPl6+urVatWmbQXAADgt5TzylCZXf1k++9Gh+W2/25UmV39LD8SVKreBr937161atXKYVlUVJTeeustSVJWVpYOHTqkfv362dd7eHiodevW2rNnj9OfZ7P9vnpvNqcZc+MG+mwN+mwN+mwN+mwuHyO9QPjJZ/vvRvkY6bpqK/e7PsOZn12pCkCpqakKCAhwWBYQEKArV67o+vXrunjxonJzc+Xv7+8wxt/fXydPnnT68/z9K/yuektqbtxAn61Bn61Bn61Bn02Sevmmqz1zLysgwLrel6oAZLW0tMsyDNfOabP99JfLjLlxA322Bn22Bn22Bn02VxXPCvK8yfpczwq68Bsh6bfk/wyLolQFoICAAKWmpjosS01NVfny5eXr6ysPDw95enoWuOA5LS2twJGjojAMmfaXwMy5cQN9tgZ9tgZ9tgZ9NkemzU9/qtG+0NNgRo32yrT5Wdr3UvUcoCZNmmjbtm0Oy7755hs1adJEkuTt7a1GjRopMTHRvj4vL0+JiYlq2rSplaUCAICfycgup5zmM2XUaO+w3KjRXjnNZykj+/dd/+OsEj0ClJGRoTNnzti/P3v2rI4cOaJKlSrp9ttv18SJE3X+/HmNHz9ektS9e3ctWbJE48ePV9euXbVt2zZ9+umnmjlzpn2O3r17a+jQoQoNDVV4eLgWLFiga9euqUuXLpbvHwAAuOHHLH+Va7ZAPka6PHMvK9ezgjJtfsrIsjb8SCUcgA4ePKiePXvav4+Pj5ckde7cWW+//bZSUlJ07tw5+/rAwEDNnDlT8fHxWrhwoWrUqKGxY8fq3nvvtY/p2LGj0tPTNXnyZKWkpCgkJERz5swp1ikwAADgWhnZ5XTVVk4BAT9d81NSpxtthsGZzl+TasIPxmaTAgIqmDI3bqDP1qDP1qDP1qDP1jGr1/nzFkWpugYIAADAFQhAAADA7RCAAACA2yEAAQAAt0MAAgAAbocABAAA3A4BCAAAuB0CEAAAcDsEIAAA4HYIQAAAwO0QgAAAgNshAAEAALdDAAIAAG6HAAQAANwOAQgAALgdAhAAAHA7BCAAAOB2CEAAAMDtEIAAAIDbIQABAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwOwQgAADgdghAAADA7RCAAACA2yEAAQAAt0MAAgAAbocABAAA3A4BCAAAuB0CEAAAcDsEIAAA4HYIQAAAwO0QgAAAgNshAAEAALdDAAIAAG6HAAQAANwOAQgAALgdAhAAAHA7BCAAAOB2CEAAAMDtEIAAAIDbIQABAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwOwQgAADgdghAAADA7RCAAACA2yEAAQAAt0MAAgAAbocABAAA3A4BCAAAuB0CEAAAcDsEIAAA4HYIQAAAwO0QgAAAgNshAAEAALdT4gFoyZIlatOmjcLCwhQXF6f9+/f/6tjs7GxNnTpVMTExCgsLU6dOnbRlyxaHMbm5uZo0aZLatGmj8PBwxcTEaNq0aTIMw+xdAQAApUSJBqCEhATFx8dr4MCBWr16tRo0aKA+ffooLS2t0PGTJk3S8uXLNWrUKCUkJKh79+4aNGiQDh8+bB8ze/ZsLV26VK+//roSEhL0yiuvaM6cOVq0aJFVuwUAAG5xJRqA5s2bp0cffVRdu3ZV/fr1NXr0aPn6+mrVqlWFjv/kk0/Uv39/RUdHKzAwUI8//riio6M1d+5c+5g9e/aobdu2uv/++3XHHXfowQcfVFRU1E2PLAEAAPdSpqQ+OCsrS4cOHVK/fv3syzw8PNS6dWvt2bOn0G2ys7Pl7e3tsMzHx0e7d++2f9+0aVOtWLFCp06dUt26dXX06FHt2rVLw4YNc7pGm83pTYo8pxlz4wb6bA36bA36bA36bB2zeu3MfCUWgC5cuKDc3Fz5+/s7LPf399fJkycL3SYqKkrz589XZGSkatWqpcTERG3atEm5ubn2MX379tWVK1fUoUMHeXp6Kjc3V4MHD1anTp2crtHfv4LT29wKc+MG+mwN+mwN+mwN+mydkux1kQJQfHx8kSccPnx4sYv5La+99ppGjhypDh06yGazKTAwUF26dHE4Zfbpp59q3bp1mjhxourXr68jR44oPj5e1apVU+fOnZ36vLS0y3L1tdM2208/cDPmxg302Rr02Rr02Rr02Tpm9Tp/3qIoUgD6+UXGN//goh97qlKlijw9PQtc8JyWlqaAgIBCt/Hz89P06dOVmZmpH3/8UdWqVdOECRMUGBhoHzN+/Hj17dtXDz30kCQpODhYP/zwg2bOnOl0ADIMmfaXwMy5cQN9tgZ9tgZ9tgZ9tk5J9rpIAciMO6i8vb3VqFEjJSYmKiYmRpKUl5enxMRE9ejR46bb+vj4qHr16srOztbGjRvVoUMH+7rr168XCGKenp7cBg8AAOyKfQ3Q6dOndebMGUVGRsrX11eGYTh1BEiSevfuraFDhyo0NFTh4eFasGCBrl27pi5dukiShgwZourVq+vll1+WJO3bt0/nz59XSEiIzp8/rylTpigvL0/PPPOMfc4///nPmjFjhm6//Xb7KbB58+apa9euxd1VAADwB+N0ALpw4YJefPFFJSUlyWazaePGjQoMDNSIESNUqVIlp+626tixo9LT0zV58mSlpKQoJCREc+bMsZ8CO3funDw8btypn5mZqUmTJik5OVlly5ZVdHS0xo8fr4oVK9rHjBw5Uu+9955Gjx6ttLQ0VatWTd26ddPAgQOd3VUAAPAHZTOcPDc0ZMgQpaWlady4cerQoYPWrl2rwMBAbd26VW+//bbWr19vVq2WS0015yLogIAKpsyNG+izNeizNeizNeizdczqdf68ReH0EaCvv/5aH3zwgWrUqOGwvE6dOvrhhx+cnQ4AAMByTj8J+urVq/L19S2w/McffyzwkEIAAIBbkdMBKCIiQmvWrHFYlpeXpzlz5qhly5auqgsAAMA0Tp8Ce/XVV9WrVy8dPHhQ2dnZ+vvf/67jx4/r4sWLWrp0qRk1AgAAuJTTASgoKEj/+te/tHjxYpUrV05Xr15Vu3bt9MQTT6hatWpm1AgAAOBSxXoOUIUKFTRgwABX1wIAAGCJYgWgixcvauXKlTpx4oQkqX79+urSpYsqV67sytoAAABM4fRF0Dt27FCbNm20aNEiXbp0SZcuXdKiRYvUtm1b7dixw4waAQAAXMrpI0BjxoxRx44d9cYbb8jT01OSlJubq9GjR2vMmDFat26dy4sEAABwJaePAJ0+fVq9e/e2hx/pp5eN9urVS6dPn3ZpcQAAAGZwOgA1bNhQJ0+eLLD85MmTatCggUuKAgAAMFORToEdPXrU/nXPnj01btw4nT59Wo0bN5b001valyxZoldeecWcKgEAAFyoSC9DbdCggWw2m35rqM1m05EjR1xWXEnjZailF322Bn22Bn22Bn22Tql5Gernn3/+uwoCAAC4lRQpANWsWdPsOgAAACxTrAchStLx48f1ww8/KDs722F527Ztf3dRAAAAZnI6ACUnJ2vgwIH67rvvHK4LstlskvSHugYIAAD8MTl9G/y4ceN0xx136JtvvpGvr6/Wr1+vxYsXKzQ0VIsWLTKjRgAAAJdyOgDt2bNHzz//vPz8/OTh4SGbzaaIiAi99NJLGjt2rBk1AgAAuJTTASgvL0/lypWTJFWpUkX/+9//JP10ofSpU6dcWx0AAIAJnL4G6K677tK3336rwMBANW7cWHPmzJGXl5dWrFihwMBAM2oEAABwKaePAA0YMEB5eXmSpOeff15nz57VE088oS+//FKvvfaaywsEAABwNaePAN177732r2vXrq0NGzboxx9/VKVKlex3ggEAANzKiv0coJ+rXLmyK6YBAACwRJEC0KBBg4o84dSpU4tdDAAAgBWKFIAqVCjai8UAAABKgyIFoPj4eLPrAAAAsIzTd4EBAACUdgQgAADgdghAAADA7RCAAACA2yEAAQAAt1OsByEmJiYqMTFRaWlp9tdi5OOOMQAAcKtzOgBNnTpV06ZNU2hoqKpWrcrrLwAAQKnjdABatmyZ4uPj9cgjj5hQDgAAgPmcvgYoOztbzZo1M6MWAAAASzgdgP76179q3bp1ZtQCAABgCadPgWVmZmrFihVKTExUcHCwypRxnGL48OEuKw4AAMAMTgegb7/9Vg0aNJAkfffddw7ruCAaAACUBk4HoEWLFplRBwAAgGV4ECIAAHA7xXoQ4oEDB/Tpp5/q3Llzys7Odlg3depUlxQGAABgFqePAK1fv16PPfaYTp48qU2bNiknJ0fHjh3Ttm3bVKFCBTNqBAAAcCmnA9CMGTM0fPhwzZgxQ15eXnrttde0YcMGdejQQbfddpsZNQIAALiU0wEoOTlZ0dHRkiRvb29dvXpVNptNvXr10ooVK1xeIAAAgKs5HYAqVqyojIwMSVK1atV07NgxSdKlS5d07do111YHAABgAqcvgo6MjNQ333yj4OBgPfjggxo3bpy2bdumb775Rq1atTKjRgAAAJdyOgCNGjVKmZmZkqQBAwbIy8tLu3fvVvv27TVgwACXFwgAAOBqTgegypUr27/28PBQ3759XVkPAACA6Yr1HCBJSktLU1pamvLy8hyW578mAwAA4FbldAA6ePCghg0bphMnTsgwDId1NptNR44ccVlxAAAAZnA6AI0YMUJ16tTRuHHj5O/vzwtQAQBAqeN0AEpOTtaUKVNUu3ZtM+oBAAAwndPPAWrVqpWOHj1qRi0AAACWcPoI0NixYzVs2DAdO3ZMd911l8qUcZyibdu2LisOAADADE4HoL1792r37t3asmVLgXVcBA0AAEqDYh0B6tSpk5577jkFBASYURMAAICpnL4G6MKFC+rVqxfhBwAAlFpOB6D27dsrKSnJjFoAAAAs4fQpsDp16mjixInatWuXgoKCClwE3bNnT5cVBwAAYAanA9BHH32ksmXLavv27dq+fbvDOpvN5nQAWrJkiT744AOlpKSoQYMGGjVqlMLDwwsdm52drZkzZ2rNmjU6f/686tatq1deeUX33Xefw7jz58/r73//u7Zu3apr166pdu3aeuuttxQWFubczgIAgD8kpwPQ5s2bXfbhCQkJio+P1+jRo9W4cWMtWLBAffr00YYNG+Tv719g/KRJk7R27VqNHTtW9erV09atWzVo0CAtW7ZMDRs2lCRdvHhRjz32mFq2bKnZs2erSpUqOn36tCpVquSyugEAQOnm9DVArjRv3jw9+uij6tq1q+rXr6/Ro0fL19dXq1atKnT8J598ov79+ys6OlqBgYF6/PHHFR0drblz59rHzJ49WzVq1FB8fLzCw8MVGBioqKgo1apVy6rdAgAAtzinjwANHz78puvj4+OLNE9WVpYOHTqkfv362Zd5eHiodevW2rNnT6HbZGdny9vb22GZj4+Pdu/ebf9+8+bNioqK0vPPP68dO3aoevXqevzxx/Xoo48WqS4AAPDH53QAunTpksP3OTk5OnbsmC5duqS77767yPNcuHBBubm5BU51+fv76+TJk4VuExUVpfnz5ysyMlK1atVSYmKiNm3apNzcXPuY5ORkLV26VL1791b//v114MABjR07Vl5eXurcubMTeyqZ8Z7X/Dl5h6y56LM16LM16LM16LN1zOq1M/M5HYCmTZtWYFleXp7eeOMNBQYGOjudU1577TWNHDlSHTp0kM1mU2BgoLp06eJwyswwDIWGhuqll16SJDVs2FDHjh3TsmXLnA5A/v4VXFq/VXPjBvpsDfpsDfpsDfpsnZLstdMBqDAeHh7q1auXevbsqWeffbZI21SpUkWenp5KS0tzWJ6WlvarD1n08/PT9OnTlZmZqR9//FHVqlXThAkTHIJX1apVdeeddzpsV69ePf3rX/9ycq+ktLTLMgynN7spm+2nH7gZc+MG+mwN+mwN+mwN+mwds3qdP29RuCQAST+desrJySnyeG9vbzVq1EiJiYmKiYmR9NORpMTERPXo0eOm2/r4+Kh69erKzs7Wxo0b1aFDB/u6Zs2a6dSpUw7jv//+e9WsWdOJvfmJYci0vwRmzo0b6LM16LM16LM16LN1SrLXTgegX17kbBiGUlJS9MUXXzh9iql3794aOnSoQkNDFR4ergULFujatWvq0qWLJGnIkCGqXr26Xn75ZUnSvn37dP78eYWEhOj8+fOaMmWK8vLy9Mwzz9jnfOqpp/TYY49pxowZ6tChg/bv368VK1ZozJgxzu4qAAD4g3I6AB0+fNjhew8PD/n5+WnYsGHq2rWrU3N17NhR6enpmjx5slJSUhQSEqI5c+bYT4GdO3dOHh437tTPzMzUpEmTlJycrLJlyyo6Olrjx49XxYoV7WPCw8M1depU/eMf/9C0adN0xx13aMSIEerUqZOzuwoAAP6gbIbBgb5fk5pqzjVAAQEVTJkbN9Bna9Bna9Bna9Bn65jV6/x5i6JEH4QIAABQEpw+BZaamqp33nlHiYmJSk9P1y8PIB05csRlxQEAAJjB6QA0bNgwnTt3Ts8995yqVatmRk0AAACmcjoA7dq1Sx9++KFCQkLMqAcAAMB0Tl8DdNtttxU47QUAAFCaOB2ARowYoYkTJ+rs2bNm1AMAAGC6Ip0Ci4yMlO1nbxi7evWq2rVrJ19fX3l5eTmM3b59u2srBAAAcLEiBaARI0aYXQcAAIBlihSAnH3FBQAAwK2MByECAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwO06/CuPq1auaNWuWtm3bprS0NOXl5Tms//zzz11WHAAAgBmcDkAjR47U9u3b9Ze//EVVq1Z1eEAiAABAaeB0ANqyZYtmzpyp5s2bm1EPAACA6Zy+BqhixYqqXLmyCaUAAABYw+kA9MILL+i9997TtWvXzKgHAADAdEU6BfbII484XOtz+vRptW7dWnfccYfKlHGcYvXq1a6tEAAAwMWKFIBiYmLMrgMAAMAyRQpAgwYNMrsOAAAAyzh9DVDbtm114cKFAssvXbqktm3buqQoAAAAMzkdgP7zn/8UePihJGVlZen8+fMuKQoAAMBMRX4O0M+f8Lx161ZVqFDB/n1eXp4SExNVs2ZN11YHAABggiIHoIEDB0qSbDabhg0b5jhJmTKqWbNmgeUAAAC3oiIHoKNHj0qS2rRpo5UrV8rPz8+0ogAAAMzk9KswNm/ebEYdAAAAlilSAFq4cKG6desmHx8fLVy48KZje/bs6ZLCAAAAzFKkADR//nzFxsbKx8dH8+fP/9VxNpuNAAQAAG55RQpAPz/txSkwAABQ2jn9HKDk5GQz6gAAALCM0xdBt2vXTjVq1FBkZKRatGihFi1aqHbt2mbUBgAAYAqnA9CXX36ppKQk7dixQ3PmzNGoUaNUrVo1RUZG6u6771ZcXJwZdQIAALiMzTAM4/dM8P3332vGjBlat26d8vLydOTIEVfVVuJSUy/r93WnIJtNCgioYMrcuIE+W4M+W4M+W4M+W8esXufPWxROHwG6du2adu3ape3bt2v79u06fPiw6tWrpyeeeEItWrRwulgAAACrOR2AIiMjVbFiRcXGxurZZ59VRESEKlWqZEZtAAAApnA6AN13333atWuX1q9fr9TUVKWmpqpFixaqW7euGfUBAAC4nNMBaPr06ZJ+ejfYjh079PXXX+u9996Tp6enWrRooYkTJ7q8SAAAAFdy+jlA+YKDg9WsWTM1adJEYWFhSk9P16effurK2gAAAEzh9BGgefPmKSkpSbt371ZGRoaCg4MVGRmpRx99VBEREWbUCAAA4FJOB6B//vOfatGihbp166aIiAhVqFC0280AAABuFU4HoFWrVplRBwAAgGWKfQ0QAABAaUUAAgAAbocABAAA3A4BCAAAuB2nA1DPnj116dKlAsuvXLminj17uqQoAAAAMzkdgLZv367s7OwCyzMzM7Vr1y6XFAUAAGCmIt8Gf/ToUfvXx48fV0pKiv37vLw8bd26VdWrV3dtdQAAACYocgB65JFHZLPZZLPZ9NRTTxVY7+vrq5EjR7q0OAAAADMUOQB9/vnnMgxDMTEx+uijj+Tn52df5+XlJX9/f3l6eppSJAAAgCsVOQDVrFlTkuOpMAAAgNLI6VdhSNL333+vpKQkpaWlKS8vz2HdoEGDXFIYAACAWZwOQCtWrNAbb7yhKlWqKCAgQDabzb7OZrMRgAAAwC3P6QD0/vvv68UXX1Tfvn3NqAcAAMB0Tj8H6OLFi+rQoYMZtQAAAFjC6QD04IMP6quvvjKjFgAAAEs4fQqsdu3aeu+997Rv3z4FBQWpTBnHKXgdBgAAuNU5HYCWL1+usmXLavv27dq+fbvDOpvNRgACAAC3PKcD0ObNm82oAwAAwDJOXwOULysrSydPnlROTs7vLmLJkiVq06aNwsLCFBcXp/379//q2OzsbE2dOlUxMTEKCwtTp06dtGXLll8dP2vWLAUHB2vcuHG/u04AAPDH4HQAunbtmkaMGKEmTZro4Ycf1rlz5yRJb775pmbNmuV0AQkJCYqPj9fAgQO1evVqNWjQQH369FFaWlqh4ydNmqTly5dr1KhRSkhIUPfu3TVo0CAdPny4wNj9+/dr2bJlCg4OdrouAADwx+V0AJo4caKOHj2qhQsXysfHx768VatWSkhIcLqAefPm6dFHH1XXrl1Vv359jR49Wr6+vlq1alWh4z/55BP1799f0dHRCgwM1OOPP67o6GjNnTvXYVxGRoZeffVVjR07VpUqVXK6LgAA8Mfl9DVAn3/+ud599101adLEYfldd92lM2fOODVXVlaWDh06pH79+tmXeXh4qHXr1tqzZ0+h22RnZ8vb29thmY+Pj3bv3u2wbMyYMYqOjlbr1q31/vvvO1VXvp895Npl8uc0Y27cQJ+tQZ+tQZ+tQZ+tY1avnZnP6QCUnp4uf3//AsuvXbvm8FqMorhw4YJyc3MLzOfv76+TJ08Wuk1UVJTmz5+vyMhI1apVS4mJidq0aZNyc3PtY9avX6/Dhw9r5cqVTtXzS/7+FX7X9iU1N26gz9agz9agz9agz9YpyV47HYBCQ0P1xRdf6Mknn3RY/tFHHxU4KmSG1157TSNHjlSHDh1ks9kUGBioLl262E+ZnTt3TuPGjdPcuXMdTtEVR1raZRmGK6q+wWb76Qduxty4gT5bgz5bgz5bgz5bx6xe589bFE4HoMGDB+vZZ5/V8ePHlZubq4ULF+rEiRPas2ePFi1a5NRcVapUkaenZ4ELntPS0hQQEFDoNn5+fpo+fboyMzP1448/qlq1apowYYICAwMlSYcOHVJaWpq6dOli3yY3N1c7duzQkiVLdODAAXl6ehapPsOQaX8JzJwbN9Bna9Bna9Bna9Bn65Rkr52+CDoiIkKffPKJcnNzFRQUpK+//lp+fn5atmyZQkNDnZrL29tbjRo1UmJion1ZXl6eEhMT1bRp05tu6+Pjo+rVqysnJ0cbN25U27ZtJUl333231q1bpzVr1tj/hIaGKjY2VmvWrCly+AEAAH9cTh8BkqRatWpp7NixLimgd+/eGjp0qEJDQxUeHq4FCxbo2rVr9iM4Q4YMUfXq1fXyyy9Lkvbt26fz588rJCRE58+f15QpU5SXl6dnnnlGklS+fHkFBQU5fEbZsmVVuXLlAssBAIB7KlYAkn46TZWWlqa8vDyH5Q0aNHBqno4dOyo9PV2TJ09WSkqKQkJCNGfOHPspsHPnzsnD48aBqszMTE2aNEnJyckqW7asoqOjNX78eFWsWLG4uwIAANyMzTCcO/t28OBBDRs2TCdOnNAvN7XZbDpy5IhLCyxJqanmXAQdEFDBlLlxA322Bn22Bn22Bn22jlm9zp+3KJw+AjRixAjVqVNH48aNk7+/v9O3vgMAAJQ0pwNQcnKypkyZotq1a5tRDwAAgOmcvgusVatWOnr0qBm1AAAAWMLpI0Bjx47VsGHDdOzYMd11110qU8Zxivzb0QEAAG5VTgegvXv3avfu3dqyZUuBdX+0i6ABAMAfU7GOAHXq1EnPPffcrz6tGQAA4Fbm9DVAFy5cUK9evQg/AACg1HI6ALVv315JSUlm1AIAAGAJp0+B1alTRxMnTtSuXbsUFBRU4CLonj17uqw4AAAAMzgdgD766COVLVtW27dv1/bt2x3W2Ww2AhAAALjlOR2ANm/ebEYdAAAAlnH6GiAAAIDSzukjQLm5ufr444+1bdu2Qt8Gv3DhQpcVBwAAYAanA9C4ceO0evVqRUdH66677uJlqAAAoNRxOgCtX79ekyZNUnR0tBn1AAAAmM7pa4C8vLxUq1YtM2oBAACwhNMB6Omnn9bChQtlGIYZ9QAAAJjO6VNgu3btUlJSkrZs2VLo2+CnTp3qsuIAAADM4HQAqlixotq1a2dGLQAAAJZwOgDFx8ebUQcAAIBleBAiAABwO04fAXrkkUcKffaPzWaTt7e3ateurc6dO+vuu+92SYEAAACu5vQRoHvvvVfJycn605/+pJYtW6ply5YqW7aszpw5o7CwMKWkpKh379767LPPzKgXAADgd3P6CNCFCxfUu3dvDRw40GH59OnT9cMPP2ju3LmaPHmypk+frpiYGJcVCgAA4CpOHwH69NNP9fDDDxdY/tBDD+nTTz+1f33q1KnfXx0AAIAJnA5APj4+2rNnT4Hle/bskY+PjyTJMAz71wAAALcap0+B9ejRQ3/729908OBBhYWFSZIOHDiglStXql+/fpKkr776SiEhIa6tFAAAwEVsRjHeabF27VotWbLEfpqrbt266tGjh2JjYyVJ169fl81mK/VHgVJTL8vVb/yw2aSAgAqmzI0b6LM16LM16LM16LN1zOp1/rxF4fQRIEnq1KmTOnXq9KvrfX19izMtAACAJXgQIgAAcDtFOgLUokULbdiwQX5+foqMjCz0QYj5tm/f7rLiAAAAzFCkADR8+HCVL19ekjRixAhTCwIAADBbkQJQ586dC/0aAACgNCpSALpy5UqRJ8w/UgQAAHCrKlIAioiIuOl1P9JPDz+02Ww6cuSISwoDAAAwS5EC0MKFC82uAwAAwDJFvgusKL777rvfVQwAAIAVivUgxJ+7cuWK1q9fr48++kiHDh3iFBgAALjlFTsA7dixQytXrtTGjRtVrVo1tWvXTq+//rorawMAADCFUwEoJSVFq1ev1sqVK3XlyhV16NBBWVlZmjZtmurXr29WjQAAAC5V5ADUv39/7dixQ/fff79GjBihe++9V56enlq2bJmZ9QEAALhckQPQli1b9OSTT+qxxx5TnTp1TCwJAADAXEV+GeqHH36ojIwMdenSRXFxcVq8eLHS09PNrA0AAMAURQ5ATZo00dixY/XVV1+pW7duWr9+ve677z7l5eXp66+/dupp0QAAACWpyAEoX9myZfXXv/5VS5cu1dq1a9W7d2/Nnj1brVu3Vv/+/c2oEQAAwKWcDkA/V69ePQ0ZMkRffvml/vGPf7iqJgAAAFP97gchSpKnp6diYmIUExPjiukAAABM9buOAAEAAJRGBCAAAOB2CEAAAMDtEIAAAIDbIQABAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwOwQgAADgdghAAADA7RCAAACA2yEAAQAAt3NLBKAlS5aoTZs2CgsLU1xcnPbv3/+rY7OzszV16lTFxMQoLCxMnTp10pYtWxzGzJw5U127dlXTpk3VqlUrPffcczp58qTZuwEAAEqJEg9ACQkJio+P18CBA7V69Wo1aNBAffr0UVpaWqHjJ02apOXLl2vUqFFKSEhQ9+7dNWjQIB0+fNg+Zvv27XriiSe0YsUKzZs3Tzk5OerTp4+uXr1q1W4BAIBbmM0wDKMkC4iLi1NYWJhef/11SVJeXp6io6P15JNPqm/fvgXGR0VFacCAAXriiSfsy/7v//5PPj4+mjBhQqGfkZ6erlatWmnx4sWKjIwscm2pqZfl6u7YbFJAQAVT5sYN9Nka9Nka9Nka9Nk6ZvU6f96iKOO6j3VeVlaWDh06pH79+tmXeXh4qHXr1tqzZ0+h22RnZ8vb29thmY+Pj3bv3v2rn3P58mVJUqVKlZyqz2ZzarhTc5oxN26gz9agz9agz9agz9Yxq9fOzFeiAejChQvKzc2Vv7+/w3J/f/9fvWYnKipK8+fPV2RkpGrVqqXExERt2rRJubm5hY7Py8vTW2+9pWbNmikoKMip+vz9i5Yii8PMuXEDfbYGfbYGfbYGfbZOSfa6RANQcbz22msaOXKkOnToIJvNpsDAQHXp0kWrVq0qdPzo0aN17Ngxffjhh05/VlqaOafA/P0rmDI3bqDP1qDP1qDP1qDP1jGr1/nzFkWJBqAqVarI09OzwAXPaWlpCggIKHQbPz8/TZ8+XZmZmfrxxx9VrVo1TZgwQYGBgQXGjhkzRl988YUWL16sGjVqOF2fYci0vwRmzo0b6LM16LM16LM16LN1SrLXJXoXmLe3txo1aqTExET7sry8PCUmJqpp06Y33dbHx0fVq1dXTk6ONm7cqLZt29rXGYahMWPGaNOmTVqwYEGh4QgAALivEj8F1rt3bw0dOlShoaEKDw/XggULdO3aNXXp0kWSNGTIEFWvXl0vv/yyJGnfvn06f/68QkJCdP78eU2ZMkV5eXl65pln7HOOHj1a//znPzV9+nSVK1dOKSkpkqQKFSrI19fX+p0EAAC3lBIPQB07dlR6eromT56slJQUhYSEaM6cOfZTYOfOnZOHx40DVZmZmZo0aZKSk5NVtmxZRUdHa/z48apYsaJ9zNKlSyVJTz75pMNnxcfH24MVAABwXyX+HKBbGc8BKr3oszXoszXoszXos3VuhecAlfiToAEAAKxGAAIAAG6HAAQAANwOAQgAALgdAhAAAHA7BCAAAOB2CEAAAMDtEIAAAIDbIQABAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwOwQgAADgdghAAADA7RCAAACA2yEAAQAAt0MAAgAAbocABAAA3A4BCAAAuB0CEAAAcDsEIAAA4HYIQAAAwO0QgAAAgNshAAEAALdDAAIAAG6nTEkXAABASSvnlSEfI11KvawqnhWVaauijOxyJV0WTEQAAgC4tcreaSqzq59s/90oSfKU9Kca7eXVfKZ+zPIv2eJgGk6BAQDcVjmvDIfwk8/2340qs6ufynlllFBlMBsBCADgtryN9ALhJ5/tvxvlbaRbXBGsQgACALgtW/bF31h/yaJKYDUCEADAbRlelX5jfUWLKoHVuAgaQLFw14w16LO5smx+8qzRvtDTYEaN9sqy+ZVAVbACAQiA07hrxhr02XwZ2eXk1XxmgQuhjRrtldN8ljKyCJt/VDbDMIySLuJWlZp6Wa7ujs0mBQRUMGVu3ECfzVPOK0N/2v3Ur/6L+VqzBRyhcAH6bK38I22euZeV61lBmTY/+msis/4fnT9vUXAECIBTinLXTIb4xfF70WdrZWSX01VbOQUEVNAF/uHkFrgIGoBTuGvGGvQZMBcBCIBTuGvGGvQZMBcBCIBTsmx+Mmq0L3Qdd824Dn0GzEUAwh9OOa8MVfFMllKTVMXzLI+yd7GM7HLKaT6zwC9n+10zXDjqEvQZMBd3gd2Eq69Od7zLgOd5mOGXtw1L+b8wuG3Y1bhrxhr02TrcPWod7gJzIzzPw3y/+VJDbht2Ke6asQZ9BszBKTAL8LZha/BSQwBAURGALMAvZmtw2zAAoKgIQBbgF7M1uG0YAFBUBCAL8IvZGtw2DAAoKgKQBfjFbA1uGwYAFBV3gVmAtw1b58csf5VrtqDgbcP0GADwMwQgi/CL2TrcNgwA+C0EIAvxixkAgFsD1wABAAC3QwACAABuhwAEAADcDgEIAAC4HQIQAABwOwQgAADgdghAAADA7RCAAACA2yEAAQAAt0MAAgAAbodXYdyEzWbenGbMjRvoszXoszXoszXos3XM6rUz89kMgzdSAQAA98IpMAAA4HYIQAAAwO0QgAAAgNshAAEAALdDAAIAAG6HAAQAANwOAQgAALgdAhAAAHA7BCAAAOB2CEAAAMDtEIBMsGTJErVp00ZhYWGKi4vT/v37bzr+008/1YMPPqiwsDDFxsbqyy+/tKjS0s2ZPq9YsUKPP/64IiMjFRkZqV69ev3mzwU/cfa/53zr169XcHCwnnvuOZMr/GNwts+XLl3S6NGjFRUVpdDQUD3wwAP8v6MInO3z/Pnz9cADDyg8PFzR0dF66623lJmZaVG1pdOOHTvUv39/RUVFKTg4WJ999tlvbpOUlKTOnTsrNDRU7dq108cff2x+oQZcav369UajRo2MlStXGseOHTNGjhxpREREGKmpqYWO37VrlxESEmLMnj3bOH78uPHuu+8ajRo1Mr799luLKy9dnO3zSy+9ZCxevNg4fPiwcfz4cWPYsGFG8+bNjf/+978WV166ONvnfMnJyca9995rPP7448aAAQMsqrb0crbPmZmZRpcuXYxnn33W2Llzp5GcnGwkJSUZR44csbjy0sXZPq9du9YIDQ011q5dayQnJxtbt2417rnnHuOtt96yuPLS5YsvvjD+8Y9/GBs3bjSCgoKMTZs23XT8mTNnjMaNGxvx8fHG8ePHjUWLFhkhISHGli1bTK2TAORif/3rX43Ro0fbv8/NzTWioqKMmTNnFjr+hRdeMPr27euwLC4uzhg1apSpdZZ2zvb5l3JycoymTZsaq1evNqnCP4bi9DknJ8fo1q2bsWLFCmPo0KEEoCJwts8ffvih0bZtWyMrK8uqEv8QnO3z6NGjjZ49ezosi4+PN7p3725qnX8kRQlA48ePNx566CGHZS+++KLx9NNPm1mawSkwF8rKytKhQ4fUunVr+zIPDw+1bt1ae/bsKXSbvXv3qlWrVg7LoqKitHfvXjNLLdWK0+dfunbtmnJyclSpUiWzyiz1itvnadOmyd/fX3FxcVaUWeoVp8+bN29WkyZNNGbMGLVu3VoPP/ywZsyYodzcXKvKLnWK0+emTZvq0KFD9tNkycnJ+vLLLxUdHW1Jze6ipH4PljF1djdz4cIF5ebmyt/f32G5v7+/Tp48Weg2qampCggIKDA+NTXVtDpLu+L0+ZcmTJigatWqOfzPEI6K0+edO3dq5cqVWrNmjQUV/jEUp8/Jycnatm2bYmNjNWvWLJ05c0ajR49WTk6OBg0aZEXZpU5x+hwbG6sLFy7o8ccfl2EYysnJUffu3dW/f38rSnYbhf0eDAgI0JUrV3T9+nX5+vqa8rkcAYLbmTVrlhISEjR16lT5+PiUdDl/GFeuXNGQIUP05ptvys/Pr6TL+UMzDEP+/v568803FRoaqo4dO6p///5atmxZSZf2h5KUlKSZM2fqb3/7mz7++GNNnTpVX375paZNm1bSpcEFOALkQlWqVJGnp6fS0tIclqelpRVIt/kCAgIKHO252XgUr8/5PvjgA82aNUvz5s1TgwYNzCyz1HO2z8nJyfrPf/6jAQMG2Jfl5eVJkho2bKgNGzaoVq1a5hZdChXnv+eqVauqTJky8vT0tC+rV6+eUlJSlJWVJW9vb1NrLo2K0+f33ntPnTp1sp/ODQ4O1tWrV/X6669rwIAB8vDgGIIrFPZ7MDU1VeXLlzft6I/EESCX8vb2VqNGjZSYmGhflpeXp8TERDVt2rTQbZo0aaJt27Y5LPvmm2/UpEkTM0st1YrTZ0maPXu2pk+frjlz5igsLMyKUks1Z/tcr149rVu3TmvWrLH/adOmjVq2bKk1a9aoRo0aVpZfahTnv+dmzZrpzJkz9oApSd9//72qVq1K+PkVxenz9evXC4Sc/NBpGIZ5xbqZEvs9aOol1m5o/fr1RmhoqPHxxx8bx48fN0aNGmVEREQYKSkphmEYxquvvmpMmDDBPn7Xrl1Gw4YNjQ8++MA4fvy4MXnyZG6DLwJn+zxz5kyjUaNGxoYNG4z//e9/9j9XrlwpqV0oFZzt8y9xF1jRONvnH374wWjatKkxZswY4+TJk8a///1vo1WrVsb06dNLahdKBWf7PHnyZKNp06bGP//5T+PMmTPGV199ZcTExBgvvPBCCe1B6XDlyhXj8OHDxuHDh42goCBj3rx5xuHDh43//Oc/hmEYxoQJE4xXX33VPj7/Nvh33nnHOH78uLF48WJLboPnFJiLdezYUenp6Zo8ebJSUlIUEhKiOXPm2A+xnjt3zuFfFM2aNdOECRM0adIk/eMf/1CdOnU0bdo0BQUFldQulArO9nnZsmXKzs7W888/7zDPoEGD9H//93+W1l6aONtnFI+zfb7tttv0wQcfKD4+Xp06dVL16tXVs2dPPfvssyW1C6WCs30eMGCAbDabJk2apPPnz8vPz09//vOfNXjw4JLahVLh4MGD6tmzp/37+Ph4SVLnzp319ttvKyUlRefOnbOvDwwM1MyZMxUfH6+FCxeqRo0aGjt2rO69915T67QZBsfxAACAe+GfbgAAwO0QgAAAgNshAAEAALdDAAIAAG6HAAQAANwOAQgAALgdAhAAAHA7BCDgD2jKlCn6y1/+UmBZ69atFRwcrM8++6yEKrth+fLlio6OVoMGDTR//vxCa/6lYcOG6bnnnrOoQvMUZV/N4Ir+ffzxx4qIiLjpmJLaP8AZPAkauMWkp6frvffe05dffqnU1FRVqlRJDRo00HPPPafmzZsXa84TJ05o6tSpmjZtmho3bqxKlSoVOi4rK0sLFizQunXrdPr0afn6+qpu3bqKi4tTp06d5OXl9Xt2ze7KlSt68803NWzYMLVv314VKlRQXl6eevTo4ZL5S0pSUpLDE3ALs3DhQouqAXAzBCDgFvN///d/ys7O1ttvv63AwEClpaUpMTFRP/74Y7HnPHPmjCSpbdu2stlshY7JyspSnz599O233+qFF15Qs2bNVL58ee3du1dz585Vw4YNFRISUuwafu6HH35Qdna2oqOjVa1aNfvycuXKuWT+ktK0aVN99dVX9u/HjRunK1eu2F8FIEmVKlXS9u3bnZ6bt7wDrsUpMOAWcunSJe3cuVOvvPKK7r77btWsWVPh4eHq16+f2rZt6zDutdde0913361mzZqpZ8+eOnr0aKFzTpkyRf3795ckNWjQQMHBwYWOW7BggXbu3Kn58+friSeeUEhIiAIDAxUbG6sVK1aodu3akn76RTx27Fi1atVKYWFheuyxx7R//377PElJSQoODlZiYqK6dOmixo0bq3v37jp58qSkn06hxMbGSpJiYmIUHByss2fPFjhtkpubq/j4eEVERKhly5YaP358gTdw5+XlaebMmWrTpo3Cw8PVqVMnbdiwoci15Nu8ebO6du2qsLAwtWzZUgMHDrSvy8rK0jvvvKN7771XTZo0UVxcnJKSkgrtobe3t6pWrWr/4+vrW2DZz0PMmjVr1KZNGzVv3lyDBw/WlStX7OuefPJJjRkzRuPGjVPLli3Vp08fSdJ3332nZ555Rk2bNlXr1q316quvKj093b7dhg0bFBsbq/DwcLVs2VK9evXS1atXHer84IMPFBUVpZYtW2r06NHKzs62r7t48aKGDBmiyMhINW7cWM8884y+//77Qvc336xZs9S6dWs1bdpUI0aMUGZm5k3HA7cCAhBwCylbtqzKli2rzz77TFlZWb867oUXXlBaWppmz56tjz/+WI0aNdJTTz1V6FGip59+2n4E4quvvnI4QvFz69atU+vWrdWwYcMC67y8vFS2bFlJ0vjx4/Wvf/1Lb7/9tlavXq3atWvrmWeeKfDZ7777roYNG6ZVq1bJ09NTI0aMkPTTCynnz58vSfroo4/01Vdf6bbbbivwmXPnztXq1av11ltv6cMPP9TFixe1adMmhzEzZ87UmjVrNHr0aK1fv169evXSq6++WuAIy6/VIklffPGFBg0apOjoaK1Zs0YLFixQeHi4ff2YMWO0Z88evfvuu1q7dq0efPDBIoWC33LmzBl9/vnnmjFjhmbOnKkdO3Zo9uzZDmNWr14tLy8vLV26VKNHj9alS5f01FNPqWHDhlq5cqXmzJmjtLQ0vfjii5Kk//3vf3r55ZfVtWtXJSQkaOHChWrXrp1DcExKStKZM2e0YMEC+89w9erV9vXDhg3TwYMH9f7772v58uUyDEN9+/Z1CEk/l5CQoClTpmjw4MFatWqVqlatqg8//PB39QawhKnvmgfgtA0bNhiRkZFGWFiY0a1bN2PixInGkSNH7Ot37NhhNGvWzMjMzHTYLiYmxli2bJlhGIYxefJko1OnTvZ1mzZtMoKCgm76ueHh4cabb7550zEZGRlGo0aNjLVr19qXZWVlGVFRUcbs2bMNwzCMbdu2GUFBQcY333xjH/PFF18YQUFBxvXr1w3DMIzDhw8bQUFBRnJysn3ML2u+55577HMahmFkZ2cb9913nzFgwADDMAwjMzPTaNy4sbF7926HGkeMGGG89NJLRa6lW7duxssvv1zo/v7nP/8xQkJCjP/+978Oy5966ilj4sSJN+2VYRjG0KFD7fX+3OTJk43GjRsbly9fti975513jLi4OPv3PXr0MB555BGH7aZNm2Y8/fTTDsvOnTtnBAUFGSdPnjQOHjxoBAUFGWfPnv3Vev785z8bOTk59mXPP/+88eKLLxqGYRinTp0ygoKCjF27dtnXp6enG+Hh4UZCQoJhGIaxatUqo3nz5vb13bp1M9544w2Hz4mLi3P4WQK3Iq4BAm4xDzzwgO6//37t3LlTe/fu1datWzVnzhyNHTtWXbp00bfffqurV6+qZcuWDttdv37dfq3Pb2natKn969jYWI0ZM6bA6aXCnDlzRtnZ2WrWrJl9mZeXl8LDw3XixAmHsT8/1Va1alVJUlpamm6//fbf/JzLly8rJSVFjRs3ti8rU6aMQkND7XWePn1a165d09NPP+2wbXZ2doFrlW5Wy5EjRxQXF1doHd99951yc3P14IMPOizPyspS5cqVf3M/bqZmzZoqX768/ftq1aopLS3NYUyjRo0cvj969KiSkpIcfn75zpw5o6ioKLVq1UqxsbGKiopSVFSUHnjgAYeL3uvXry9PT0/791WrVtV3330n6aeL5cuUKePQ9ypVqqhu3boFfr75Tpw4oe7duzssa9Kkya+eJgRuFQQg4Bbk4+Oje+65R/fcc48GDhyo1157TVOmTFGXLl2UkZGhqlWratGiRQW2q1ChQpHmX7Nmjf3r/F/CderU0alTp1xSv/RTYMmXf+F1Xl6ey+bPv65l5syZql69usO6X14sfLNafH19b/oZnp6e9lNnP5d/SrC4fl5Tvl+G0D/96U8F6vnzn/+sV155pcC2VatWlaenp+bNm6fdu3fr66+/1qJFi/Tuu+9qxYoVCgwMLPRzbTZbkcIv8EfDNUBAKVC/fn37L/xGjRopNTVVnp6eql27tsMfPz+/Is338238/f0lSQ8//LC++eYbHT58uMD47OxsXb16VbVq1ZKXl5d2797tsO7AgQOqX7++C/b0JxUqVFDVqlW1b98++7KcnBwdOnTI/v2dd94pb29v/fDDDwX6UNg1Rb8mKChIiYmJha4LCQlRbm6u0tPTC3xG/pEkKzVq1EjHjh1TzZo1C9STH8hsNpuaN2+u559/XmvWrJGXl1eRn/t05513Kicnx6HvFy5c0KlTp37153vnnXc6jJdU4HvgVkQAAm4hFy5cUM+ePfXJJ5/o6NGjSk5O1qeffqo5c+bY7wJr3bq1mjRpooEDB+qrr77S2bNntXv3br377rs6cOBAsT+7V69eatasmXr16qUlS5bYPz8hIUHdunXT6dOnVbZsWT322GMaP368tmzZouPHj2vUqFG6fv26/vrXv7qqDZKknj17avbs2frss8904sQJ+0XA+cqXL2+/wHv16tU6c+aMDh06pEWLFjlc1PtbBg0apPXr12vy5Mk6ceKEvv32W82aNUuSVLduXcXGxmrIkCHauHGjkpOTtX//fs2cOVNffPGFS/e3KB5//HFdvHhRL730kvbv368zZ85o69atGj58uHJzc7Vv3z7NmDFDBw4c0A8//KCNGzcqPT1d9erVK9L8derUUdu2bTVq1Cjt3LlTR48e1auvvqrq1as73IX4cz179tSqVau0atUqnTp1SpMnT9axY8dcuduAKTgFBtxCypUrp8aNG2vBggU6c+aMcnJyVKNGDcXFxdlvZbfZbJo1a5YmTZqk4cOH68KFCwoICFBERIQCAgKK/dne3t6aN2+e5s+fr2XLlumdd97Rn/70J9WrV09PPvmk7rrrLknSK6+8IsMwNGTIEGVkZCg0NFRz5sz51YcrFtfTTz+tlJQUDR06VB4eHuratavatWuny5cv28e8+OKL8vPz08yZM3X27FlVqFBBDRs2tPeqKFq2bKn33ntP06dP16xZs1S+fHlFRkba18fHx+v999/X22+/rf/973+qXLmymjRpovvvv9+Vu1sk1atX19KlSzVhwgT16dNHWVlZuv3223XvvffKw8ND5cuX144dO7RgwQJduXJFt99+u4YNG6bo6Ogif0Z8fLzGjRun/v37Kzs7WxEREZo1a9avPgSzY8eOOnPmjP7+978rMzNTDzzwgB577LFfvdsQuFXYDE7+AgAAN8MpMAAA4HYIQAAAwO0QgAAAgNshAAEAALdDAAIAAG6HAAQAANwOAQgAALgdAhAAAHA7BCAAAOB2CEAAAMDtEIAAAIDbIQABAAC38/+TUXawnmajuwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "x = [0.8, 0.82, 0.84, 0.86, 0.88, 0.9, 0.92, 0.94, 0.96]\n",
        "y = [calculate_alignment(i, data) for i in x]\n",
        "\n",
        "plt.scatter(x, y, s=80, facecolors='none', edgecolors='r')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "-FYfmfNwB8-8",
        "outputId": "ab3ed221-6158-4baa-e9e3-2030b7a57452"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1088\n",
            "1070\n",
            "1029\n",
            "990\n",
            "895\n",
            "724\n",
            "304\n",
            "100\n",
            "40\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-26-d77db73b6074>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-26-d77db73b6074>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-26-d77db73b6074>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-26-d77db73b6074>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAys0lEQVR4nO3de1xVdb7/8TdgsCluowgKYio5WmnY8UJmY1YUZlPemrHJEpl+OhpUxmkcnTjq6BTNzdG8Zc1oZt7yeCmt6Bil5YyXRJ0y0ywaNQTUSjBMRPb6/bHGXVsR90Y2sL++no/HehCL71rr+5Eea79Z67u+K8CyLEsAAAB+LrChOwAAAFAXCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACM0aegO1Ben06lDhw4pPDxcAQEBDd0dAADgAcuydPz4ccXFxSkwsOZrMZdMqDl06JASEhIauhsAAKAWDh48qFatWtXY5pIJNeHh4ZLsf5SIiIgG7g0AAPBEWVmZEhISXJ/jNblkQs2ZW04RERGEGgAA/IwnQ0cYKAwAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYIRL5pFuAAAaxP790oYN0rffSpGR0u23SzExDd0rIxFqAADwhU2bpKefll5/XbIsqUkT6fRp6bLLpHvvlbKzpWuuaeheGoXbTwAA1LWlS6Xeve2rNHPnSmVlUmWldOSI9Mwz0ubN0g03SOvXN3RPjVKrUDNr1iy1adNGDodDycnJ2rp163nbVlZWavLkyUpMTJTD4VBSUpJyc3Pd2syZM0fXXXeda7bfnj176s0333Rrc/LkSWVkZKhZs2YKCwvT4MGDVVJSUpvuAwDgO//4h/Tgg9J990n5+dKIEdKZKf6jo6WsLOnDD6XkZKl/f2nfvobtr0G8DjXLli1TVlaWJk6cqO3btyspKUmpqak6fPhwte2zs7M1d+5czZgxQ7t379aoUaM0cOBA7dixw9WmVatWeuaZZ5Sfn69t27bp1ltvVf/+/fXxxx+72jz++ONas2aNli9frg0bNujQoUMaNGhQLUoGAMCHpkyROneW5s2zbzVVJyxMWrnS/vqXv9Rv/0xmealHjx5WRkaG6/uqqiorLi7OysnJqbZ9y5YtrZkzZ7qtGzRokDV06NAaj/OjH/3I+tvf/mZZlmUdO3bMuuyyy6zly5e7fv7JJ59YkqxNmzZ51O/S0lJLklVaWupRewAAvLZvn2VJlvXii561nzTJsq64wrKOHfNtv/yYN5/fXl2pOXXqlPLz85WSkuJaFxgYqJSUFG3atKnabSoqKuRwONzWhYaGauPGjdW2r6qq0tKlS1VeXq6ePXtKkvLz81VZWel23I4dO6p169Y1HresrMxtAQDAp/LypKAg6ec/96z9/fdL5eX2GBtcNK9CzdGjR1VVVaXY2Fi39bGxsSouLq52m9TUVE2dOlX79u2T0+nUunXrtHLlShUVFbm1++ijjxQWFqaQkBCNGjVKq1at0jX/GRVeXFys4OBgRUVFeXzcnJwcRUZGupaEhARvSgUAwHvHj9u3lEJDPWsfHW1/5Q/vOuHzp5+mT5+u9u3bq2PHjgoODlZmZqbS09MVGOh+6A4dOmjnzp3asmWLRo8erbS0NO3evbvWxx0/frxKS0tdy8GDBy+2FAAAahYZaQeb8nLP2p8ZjxoZ6bs+XUK8CjXR0dEKCgo656mjkpIStWjRotptmjdvrtWrV6u8vFz79+/Xnj17FBYWpnbt2rm1Cw4O1lVXXaWuXbsqJydHSUlJmj59uiSpRYsWOnXqlI4dO+bxcUNCQlxPU51ZAADwqdtvt+ekWbrUs/YLF9pPRt14o2/7dYnwKtQEBwera9euysvLc61zOp3Ky8tzjX85H4fDofj4eJ0+fVorVqxQ//79a2zvdDpVUVEhSeratasuu+wyt+Pu3btXBw4cuOBxAQCoN23aSP36SdOmSf/5DDuvb76RXnhBSkuzb1nhonk9o3BWVpbS0tLUrVs39ejRQ9OmTVN5ebnS09MlScOGDVN8fLxycnIkSVu2bFFhYaG6dOmiwsJCTZo0SU6nU2PHjnXtc/z48brzzjvVunVrHT9+XIsXL9b69ev11ltvSZIiIyP10EMPKSsrS02bNlVERIQeeeQR9ezZUzfccENd/DsAAFA3JkywJ94bOlRatEgKCTm3zbFj0j332DMMZ2XVexdN5XWoGTJkiI4cOaIJEyaouLhYXbp0UW5urmvw8IEDB9zGy5w8eVLZ2dkqKChQWFiY+vXrp4ULF7oN+j18+LCGDRumoqIiRUZG6rrrrtNbb72l22+/3dXmr3/9qwIDAzV48GBVVFQoNTVVs2fPvojSAQDwgR49pGXL7Mn3OnWSHn7Y/u+mTaWiIumll+xZhr/7TnrjDalt24busTECLMuyGroT9aGsrEyRkZEqLS1lfA0AwPd27pT++Efpf//XfkXCGZdfLj3wgDR2rJSY2GDd8xfefH4TagAA8KWSEmnjRvst3VFRUp8+PO3kBW8+v3lLNwAAvhQbKw0e3NC9uCTwlm4AAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGKFJQ3cAAAD4sU8+kZYskYqKpOBgqUsX6Re/kMLC6r0rXKkBAADe27VLuu026ZprpFmzpJ07pfffl0aNkuLipF//Wjp1ql67RKgBAADe2bJF6tVLKi6WFi+WDh2SPvhA+vBD6YsvpEcflZ59VrrrLqmiot66VatQM2vWLLVp00YOh0PJycnaunXredtWVlZq8uTJSkxMlMPhUFJSknJzc93a5OTkqHv37goPD1dMTIwGDBigvXv3urXp06ePAgIC3JZRo0bVpvsAAKC2vvlGuvtuqVMnafNm+1ZTSMj3P2/dWvr976W33rKv3Pz3f9db17wONcuWLVNWVpYmTpyo7du3KykpSampqTp8+HC17bOzszV37lzNmDFDu3fv1qhRozRw4EDt2LHD1WbDhg3KyMjQ5s2btW7dOlVWVuqOO+5QeXm5275GjBihoqIi1/LHP/7R2+4DAICLsWCBHWyWL5fCw8/frk8faeJE6e9/l776ql66FmBZluXNBsnJyerevbtmzpwpSXI6nUpISNAjjzyicePGndM+Li5OTz75pDIyMlzrBg8erNDQUL388svVHuPIkSOKiYnRhg0b1Lt3b0n2lZouXbpo2rRp3nTXpaysTJGRkSotLVVERESt9gEAwCXNsqSOHaXrr5eWLr1w+yNHpFatpKeekp54olaH9Obz26srNadOnVJ+fr5SUlK+30FgoFJSUrRp06Zqt6moqJDD4XBbFxoaqo0bN573OKWlpZKkpk2buq1ftGiRoqOj1alTJ40fP14nTpw47z4qKipUVlbmtgAAgItw/Lj06afSPfd41r55c+nGG6Vt23zbr//w6pHuo0ePqqqqSrGxsW7rY2NjtWfPnmq3SU1N1dSpU9W7d28lJiYqLy9PK1euVFVVVbXtnU6nxowZo169eqlTp06u9ffff7+uvPJKxcXF6cMPP9RvfvMb7d27VytXrqx2Pzk5Ofrd737nTXkAAKAmJ0/aX6+4wvNtLr9c+u473/TnLD6fp2b69OkaMWKEOnbsqICAACUmJio9PV3z5s2rtn1GRoZ27dp1zpWckSNHuv67c+fOatmypW677TZ9/vnnSkxMPGc/48ePV1ZWluv7srIyJSQk1FFVAABcgqKipKAgqaDAs/aWZbft1cun3TrDq9tP0dHRCgoKUklJidv6kpIStWjRotptmjdvrtWrV6u8vFz79+/Xnj17FBYWpnbt2p3TNjMzU2vXrtW7776rVq1a1diX5ORkSdJnn31W7c9DQkIUERHhtgAAgIsQHCwNGCD97W92YLmQf/5T2rNH+vnPfd41yctQExwcrK5duyovL8+1zul0Ki8vTz179qxxW4fDofj4eJ0+fVorVqxQ//79XT+zLEuZmZlatWqV3nnnHbVt2/aCfdm5c6ckqWXLlt6UAAAALsbDD0u7d0uvvFJzu6oq6Xe/k9q3l34wFteXvL79lJWVpbS0NHXr1k09evTQtGnTVF5ervT0dEnSsGHDFB8fr5ycHEnSli1bVFhYqC5duqiwsFCTJk2S0+nU2LFjXfvMyMjQ4sWL9eqrryo8PFzFxcWSpMjISIWGhurzzz/X4sWL1a9fPzVr1kwffvihHn/8cfXu3VvXXXddXfw7AAAAT9xyi33lZfhw+8rNwIHntvnuO2nECOmdd6S1a6XA+pnr1+tQM2TIEB05ckQTJkxQcXGxunTpotzcXNfg4QMHDijwB50/efKksrOzVVBQoLCwMPXr108LFy5UVFSUq82cOXMk2Y9t/9D8+fM1fPhwBQcH6+2333YFqISEBA0ePFjZ2dm1KBkAANRaQIA9V80DD0iDBkk9ekgjR0pXX23PHvzmm9K8efaTUosXS3371l/XvJ2nxl8xTw0AAHXI6ZTWrJHmzLFnDz4jKkpKT5dGj7ZvPV0kbz6/eUs3AADwXmCg1L+/vRw5Ih0+bN+OSkiQzpqfrr4QagAAwMVp3txeGhhv6QYAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwQq1CzaxZs9SmTRs5HA4lJydr69at521bWVmpyZMnKzExUQ6HQ0lJScrNzXVrk5OTo+7duys8PFwxMTEaMGCA9u7d69bm5MmTysjIULNmzRQWFqbBgwerpKSkNt0HAAAG8jrULFu2TFlZWZo4caK2b9+upKQkpaam6vDhw9W2z87O1ty5czVjxgzt3r1bo0aN0sCBA7Vjxw5Xmw0bNigjI0ObN2/WunXrVFlZqTvuuEPl5eWuNo8//rjWrFmj5cuXa8OGDTp06JAGDRpUi5IBAICJAizLsrzZIDk5Wd27d9fMmTMlSU6nUwkJCXrkkUc0bty4c9rHxcXpySefVEZGhmvd4MGDFRoaqpdffrnaYxw5ckQxMTHasGGDevfurdLSUjVv3lyLFy/WvffeK0nas2ePrr76am3atEk33HDDBftdVlamyMhIlZaWKiIiwpuSAQBAA/Hm89urKzWnTp1Sfn6+UlJSvt9BYKBSUlK0adOmarepqKiQw+FwWxcaGqqNGzee9zilpaWSpKZNm0qS8vPzVVlZ6Xbcjh07qnXr1jUet6yszG0BAADm8irUHD16VFVVVYqNjXVbHxsbq+Li4mq3SU1N1dSpU7Vv3z45nU6tW7dOK1euVFFRUbXtnU6nxowZo169eqlTp06SpOLiYgUHBysqKsrj4+bk5CgyMtK1JCQkeFMqAADwMz5/+mn69Olq3769OnbsqODgYGVmZio9PV2BgdUfOiMjQ7t27dLSpUsv6rjjx49XaWmpazl48OBF7Q8AADRuXoWa6OhoBQUFnfPUUUlJiVq0aFHtNs2bN9fq1atVXl6u/fv3a8+ePQoLC1O7du3OaZuZmam1a9fq3XffVatWrVzrW7RooVOnTunYsWMeHzckJEQRERFuCwCgEfnuO2n+fOnGG6VmzaQf/UhKSpKmTZO++aahewc/5FWoCQ4OVteuXZWXl+da53Q6lZeXp549e9a4rcPhUHx8vE6fPq0VK1aof//+rp9ZlqXMzEytWrVK77zzjtq2beu2bdeuXXXZZZe5HXfv3r06cODABY8LAGiE3n9fuvJK6aGHpKgo6YknpPHjpauvlsaOlVq1ki7yij0uPU283SArK0tpaWnq1q2bevTooWnTpqm8vFzp6emSpGHDhik+Pl45OTmSpC1btqiwsFBdunRRYWGhJk2aJKfTqbFjx7r2mZGRocWLF+vVV19VeHi4a5xMZGSkQkNDFRkZqYceekhZWVlq2rSpIiIi9Mgjj6hnz54ePfkEAGhE/vEP6fbb7Ss0L7wgJSa6/7y42A45v/iFZFn2V8ADXoeaIUOG6MiRI5owYYKKi4vVpUsX5ebmugYPHzhwwG28zMmTJ5Wdna2CggKFhYWpX79+Wrhwodug3zlz5kiS+vTp43as+fPna/jw4ZKkv/71rwoMDNTgwYNVUVGh1NRUzZ4929vuAwAaUlWVNHSo1KOH9OabUkjIuW1atJAWLpQCA+0rOamp0n+ehgVq4vU8Nf6KeWoAoBFYs0a65x5p2zapa9ea2x4+LCUkSDk5UlZW/fQPjY7P5qkBAOCizJ8vde9+4UAjSTEx0r33SvPm+b5fMAKhBgBQf/79b88CzRldu9rbAB4g1AAA6k9goD3411OWZW8DeID/UwAA9ad9e2njRs+DzcaN9jaABwg1AID6M2KE9PHH9jw1F3LwoPTaa9LIkb7vF4xAqAEA1J9bbpGuvVbKzJT+8/LialVWSr/6lRQRYT8CDniAUAMAqD8BAdKyZfZVmN69q78V9dFHUr9+0ttvS6+8IoWFNUxf4Xe8nnwPAICLcu21dpi5917pJz+ROne2A05QkJSfb8843LKllJsr3XprQ/cWfoRQAwCof9dea4+tycuTnn/eHmPjdEqtW9tXZwYMkC67rKF7CT9DqAEANIzAQPsdULff3tA9gSEYUwMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYIRahZpZs2apTZs2cjgcSk5O1tatW8/btrKyUpMnT1ZiYqIcDoeSkpKUm5vr1ua9997T3Xffrbi4OAUEBGj16tXn7Gf48OEKCAhwW/r27Vub7gMAAAN5HWqWLVumrKwsTZw4Udu3b1dSUpJSU1N1+PDhattnZ2dr7ty5mjFjhnbv3q1Ro0Zp4MCB2rFjh6tNeXm5kpKSNGvWrBqP3bdvXxUVFbmWJUuWeNt9AABgqADLsixvNkhOTlb37t01c+ZMSZLT6VRCQoIeeeQRjRs37pz2cXFxevLJJ5WRkeFaN3jwYIWGhurll18+t0MBAVq1apUGDBjgtn748OE6duxYtVdxPFFWVqbIyEiVlpYqIiKiVvsAAAD1y5vPb6+u1Jw6dUr5+flKSUn5fgeBgUpJSdGmTZuq3aaiokIOh8NtXWhoqDZu3OjNoSVJ69evV0xMjDp06KDRo0frq6++Om/biooKlZWVuS0AAMBcXoWao0ePqqqqSrGxsW7rY2NjVVxcXO02qampmjp1qvbt2yen06l169Zp5cqVKioq8qqjffv21UsvvaS8vDz94Q9/0IYNG3TnnXeqqqqq2vY5OTmKjIx0LQkJCV4dDwAA+BefP/00ffp0tW/fXh07dlRwcLAyMzOVnp6uwEDvDn3ffffpnnvuUefOnTVgwACtXbtWH3zwgdavX19t+/Hjx6u0tNS1HDx4sA6qAQAAjZVXySI6OlpBQUEqKSlxW19SUqIWLVpUu03z5s21evVqlZeXa//+/dqzZ4/CwsLUrl272vdaUrt27RQdHa3PPvus2p+HhIQoIiLCbQEAAObyKtQEBwera9euysvLc61zOp3Ky8tTz549a9zW4XAoPj5ep0+f1ooVK9S/f//a9fg/vvzyS3311Vdq2bLlRe0HAACYoYm3G2RlZSktLU3dunVTjx49NG3aNJWXlys9PV2SNGzYMMXHxysnJ0eStGXLFhUWFqpLly4qLCzUpEmT5HQ6NXbsWNc+v/32W7crLl988YV27typpk2bqnXr1vr222/1u9/9ToMHD1aLFi30+eefa+zYsbrqqquUmpp6sf8GAADAAF6HmiFDhujIkSOaMGGCiouL1aVLF+Xm5roGDx84cMBtvMzJkyeVnZ2tgoIChYWFqV+/flq4cKGioqJcbbZt26ZbbrnF9X1WVpYkKS0tTS+++KKCgoL04YcfasGCBTp27Jji4uJ0xx13aMqUKQoJCalt7QAAwCBez1Pjr5inBgAA/+OzeWoAAAAaK0INAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQAAMAKhBgAAGIFQAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABiBUAMAAIxAqAEAAEYg1AAAACMQagAAgBEINQAAwAiEGgAAYARCDQD4A8tq6B4AjR6hBgAaq48/ljIzpZgYKShIuvxyqU8fadky6dSphu4d0Og0aegOAADOUlUljRkjzZwpxcZK6elSu3ZSebn06qvSffdJV10lvf669OMfN3RvgUaDUAMAjYllSaNHS/PmSdOnS6NGScHB3/88K0v68EM72Nx8s7R5s3TllQ3XX6AR4fYTADQm69ZJL7xgL48+6h5ozrjuOundd6WQELsNAEmEGgBoXGbPlpKSpOHDa24XGys9+aS0dq20f3+9dA1o7Ag1ANBYfPWVtGaNfcspIODC7e+/X7riCunll33fN8APEGoAoLEoKpKcTvtKjSeuuMIeMHzwoG/7BfgJQg0ANBZBQfbXqirPt6mqkprwzAcgEWoAoPFISLDnolm/3rP2xcXSJ59IHTv6tFuAvyDUAEBjERYmDR0qzZ0rnT594fZ//7t9leaBB3zfN8APEGoAoDHJyJAOHZJ+/euaX42wY4f0zDNSWpoUFVVv3QMaM0INADQmSUnSs89K06ZJw4ZJ+/a5//y77+wrNH36SFdfLf35zw3RS6BRYnQZADQ2GRn2raisLPtx7T59pMRE+zUJb70lffONdO+9dri54oqG7i3QaHClBgAao7Q06csvpRdftIPLRx/Zj3w/9JB99Wb5cikioqF7CTQqAZZ1abzPvqysTJGRkSotLVUEJwIAAPyCN5/fXKkBAABGINQAAAAjEGoAAIARCDUAAMAIhBoAAGAEQg0AADACoQYAABihVqFm1qxZatOmjRwOh5KTk7V169bztq2srNTkyZOVmJgoh8OhpKQk5ebmurV57733dPfddysuLk4BAQFavXr1OfuxLEsTJkxQy5YtFRoaqpSUFO07e/pwAABwyfI61CxbtkxZWVmaOHGitm/frqSkJKWmpurw4cPVts/OztbcuXM1Y8YM7d69W6NGjdLAgQO1Y8cOV5vy8nIlJSVp1qxZ5z3uH//4Rz377LN67rnntGXLFl1xxRVKTU3VyZMnvS2h7uzeLY0bZ79VNy3NfrlccXHD9QcAgEuZ5aUePXpYGRkZru+rqqqsuLg4Kycnp9r2LVu2tGbOnOm2btCgQdbQoUOrbS/JWrVqlds6p9NptWjRwvrTn/7kWnfs2DErJCTEWrJkiUf9Li0ttSRZpaWlHrWv0Z49ltWnj2VJltW8uWXdfLNl9eplWaGhltWkiWUNHWpZ33xz8ccBAOAS583nt1dXak6dOqX8/HylpKS41gUGBiolJUWbNm2qdpuKigo5HA63daGhodq4caPHx/3iiy9UXFzsdtzIyEglJyfXeNyysjK3pU58+KF04432O1iWLLHfzbJ+vbRxo3TokPSnP0lvvCH17m2/dM5fWZb0zjvSz34mtWsnxcdL119vX406z1U5AAAakleh5ujRo6qqqlJsbKzb+tjYWBWf57ZLamqqpk6dqn379snpdGrdunVauXKlioqKPD7umX17c9ycnBxFRka6loSEBI+Pd14nT0o//al05ZXSpk3SffdJwcHf/zwqShoz5vuAk5Z28cdsCAcPSt27S7fdJn38sR1sRoyQrrlGmjRJSkiQ/vIXO/gAANBI+Pzpp+nTp6t9+/bq2LGjgoODlZmZqfT0dAUG+vbQ48ePV2lpqWs5ePDgxe90+XL7A3/JEulHPzp/u2uukf76V2nNGmnv3os/bn0qKpJ+8hPp6FEpL88ONX/4gx1mFi2SCgulzEzpiSekp59u6N4CAODiVbKIjo5WUFCQSkpK3NaXlJSoRYsW1W7TvHlzrV69WuXl5dq/f7/27NmjsLAwtWvXzuPjntm3N8cNCQlRRESE23LR5syRbr9d6tDhwm1/9jMpOlp67rmLP259ysiQKiul99+Xbr1VCghw/3mzZvZVmkmTpOxsKT+/QboJAMDZvAo1wcHB6tq1q/Ly8lzrnE6n8vLy1LNnzxq3dTgcio+P1+nTp7VixQr179/f4+O2bdtWLVq0cDtuWVmZtmzZcsHj1qnt26W77vKsrcMhpaRIP3jKq9E7cEB69VVpwgT7FlNNnnzSbjN7dv30DQCAC/D6HlBWVpZeeOEFLViwQJ988olGjx6t8vJypaenS5KGDRum8ePHu9pv2bJFK1euVEFBgd5//3317dtXTqdTY8eOdbX59ttvtXPnTu3cuVOSPTB4586dOnDggCQpICBAY8aM0e9//3u99tpr+uijjzRs2DDFxcVpwIABF1G+FyxLOnXKDiueCg2VKip816e6tmCBdMUV9iPqF9KkifSrX0mLF0snTvi+bwAAXEATbzcYMmSIjhw5ogkTJqi4uFhdunRRbm6uaxDvgQMH3MbLnDx5UtnZ2SooKFBYWJj69eunhQsXKioqytVm27ZtuuWWW1zfZ2VlSZLS0tL04osvSpLGjh2r8vJyjRw5UseOHdNNN92k3Nzcc56s8pmAACkmRvJmwr99++xt/EVBgdSpkxQW5ln7G26wB0+XlEht2/q2b77idEq7dklff20HumuvlS6/vKF7BQCohQDLujQeYSkrK1NkZKRKS0trP75mzBh7sOyXX0ohITW33b3b/oBcssR+SsofpKfbA5v/+U/P2r/9tj3GqKDA/0JNaan0/PP2mKeCgu/XR0VJw4dLjz7qfzUBgIG8+fzm3U/eGD3afipo+vSa2zmd9riUmBhp0KD66VtdSEyUPvpIOn7cs/abN9u32M561L7R279fSk62Bzr36mU/5fXpp9IHH0gjR0oLF0pJSfZ6AIDfINR4o0MH6de/tl+N8Oyzdng523ff2R+MK1ZIs2a5z2PT2KWl2eNjFi26cNvTp6W5c+3xN/50u+bYMSk11R4ftWuX9NJL9lNe7dtL3brZj69/8YU9weI99/jXQG8AuMR5PabmkvfMM/YH+mOP2cFm5Ej7r/qqKundd6V586SyMnvQ7b33NnRvvZOQIA0YIE2eLPXrJ7Vuff62U6bYt+EefrjeulcnnnvOvlLz0UfSVVdV3yY8XFq50p6AMDtbev31+u0jAKBWGFNTW5s22Y8zv/KK/Ve/ZE/I98tfSqNGnf8Ds7ErLpZ69rRD2t//bj+W/sO5ao4ckZ56yr4F9/TT0g+edGv0qqrsW2y33mqHzwuZN0/6f/9P+uwz+1URAIB6583nN6HmYp04YX/QN2lij6G57LK623dDKSy0xwJt3Sr9+Mf2qyFCQ+0P91Wr7JDzzDP2wGl/sm2bffXlvffsWZMv5MQJe7LBnBz/qxUADOHN5ze3ny7W5Zfb74IySXy8PQj4/fftWZTXrLEf3Y6Jsa/SpKfbH/b+5uuv7a+evgfs8svtWaHPbAcAaNQINaheQID9pvHevRu6J3XnzIBmT9/Y7nTaT4L500Do86mstL+acCURAM6Dp59w6UhKsicWXLHCs/Z5efZ8Nr16+bZfvvLll9LEiVKrVvZTeMHB9uDvyZPtF5cCgGEINbh0hIdLDz4ovfCC/eh9TSxLmjHDnmH5ppvqp391ad48e3Dz1KnS3XdL8+fb6/r2tR9bb9PGfsUFABiEUINLy2OP2Vdf7r///O/lsix7IPSaNdJvf3vum8obu5dekh56yB77dOiQPS5q+HD7++eft9f94hf2HEOvvNLQvQWAOkOowaWlQwf7g/zNN+1H1xcv/j7cWJY919DAgXaYmTjR/vD3J8eO2TNfDx9uz8kTHn5um8hI+6rNkCH2PEvl5fXdSwDwCUINLj133SWtX2/PKzR0qP2EU2Ki/XTXrbfar0x4+WVp0qSG7qn3Fiyw503Kyan5ClNgoH0bqqyM21AAjME8Nbi0ffKJ9Npr37+l+yc/kfr08b9bTmd062a/iHP5cs/a//Sn9u2499/3bb8AoJaYpwbw1NVX24spCgvtgcGe6txZWrbMd/0BgHrE7SfAJE2a2O8m81RlJXPXADAGoQYwybXX2vPreMKypLfftrcBAAMQagCT/OpX9stWd+y4cNt//lP617/sbQDAAAwUBkxy+rT92HpEhP2EV2Rk9e2+/toeFG1Z0q5d9tNQANAIefP5zZkMMEmTJtLq1dK//23PhPzmm/Y7rM6oqpLWrpVuvFE6fNh+6zqBBoAhePoJME3nztI//mG/EqJfP/t1Cd262Vdltm6V9u+XkpPtGZPbt2/o3gJAneFPNMBE11wjbdtmj6+57Tbpq6+kb76x3/20dau0eTOBBoBxuFIDmCogQLrhBnsBgEsAoQaAf9u7174ideKE1KyZdMcd9iswAFxyCDUA/NO6dfb7q87My3Nm4sHQUPst7NnZUps2DdpFAPWLMTUA/M/MmVJqqvTtt/bLR0+csF/kWVhov2H9jTekHj08m68HgDGYpwaAf3n1VWnAAGnMGOkvf6n+kfSjR6U775S+/FLauVOKja3nTgKoK8xTA8BMliVNnCjdfrs0der559iJjpZef92+kjNnTv32EUCDIdQA8B+bNtmvdvjv/7af7qpJTIw9V8/zz9sv7gRgPEINAP+xbp39hNPtt3vW/v77paIiafdu3/YLQKNAqAHgP44ft0ONp692iI62v5aV+a5PABoNQg0A/xEZaQ8CrqryrP3hw99vB8B4hBoA/qNvX/sN42++6Vn7hQulhAT7tREAjEeoAeA/une3X875l7+4v328OoWF0qJF0siR9sR8AIxHqAHgX6ZMkd57Txo9+vy3oYqK7HlqoqOlUaPqt38AGgyhBoB/6dtX+tvfpL//Xbr+emnuXHucTUWF9Omn0rhxUufO9m2q3NzvBwsDMB6hBoD/SU+XNmyQ2rWTHn5Yat5ccjikDh2k556z56fZupWxNMAlhhvNAPxTr172cuCAHWBOnJCaNpVuuUW64oqG7h2ABkCoAeDfWre2FwCXPG4/AQAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMcMnMU2NZliSprKysgXsCAAA8deZz+8zneE0umVBz/PhxSVJCQkID9wQAAHjr+PHjioyMrLFNgOVJ9DGA0+nUoUOHFB4eroCAgDrdd1lZmRISEnTw4EFFRETU6b4bA9Prk8yvkfr8n+k1Up//81WNlmXp+PHjiouLU2BgzaNmLpkrNYGBgWrVqpVPjxEREWHs/6yS+fVJ5tdIff7P9Bqpz//5osYLXaE5g4HCAADACIQaAABgBEJNHQgJCdHEiRMVEhLS0F3xCdPrk8yvkfr8n+k1Up//aww1XjIDhQEAgNm4UgMAAIxAqAEAAEYg1AAAACMQagAAgBEINdWYNWuW2rRpI4fDoeTkZG3durXG9tOmTVOHDh0UGhqqhIQEPf744zp58uRF7dPX6rrGnJwcde/eXeHh4YqJidGAAQO0d+9eX5dxXr74HZ7xzDPPKCAgQGPGjPFBzz3ji/oKCwv1wAMPqFmzZgoNDVXnzp21bds2X5ZRo7qusaqqSv/zP/+jtm3bKjQ0VImJiZoyZYpH75PxBW/qq6ys1OTJk5WYmCiHw6GkpCTl5uZe1D7rQ13X6M/nGU9/h2f423nG0/p8fp6x4Gbp0qVWcHCwNW/ePOvjjz+2RowYYUVFRVklJSXVtl+0aJEVEhJiLVq0yPriiy+st956y2rZsqX1+OOP13qfvuaLGlNTU6358+dbu3btsnbu3Gn169fPat26tfXtt9/WV1kuvqjvjK1bt1pt2rSxrrvuOuuxxx7zcSXV80V9X3/9tXXllVdaw4cPt7Zs2WIVFBRYb731lvXZZ5/VV1lufFHjU089ZTVr1sxau3at9cUXX1jLly+3wsLCrOnTp9dXWS7e1jd27FgrLi7Oev31163PP//cmj17tuVwOKzt27fXep++5osa/fk840l9Z/jjecaT+urjPEOoOUuPHj2sjIwM1/dVVVVWXFyclZOTU237jIwM69Zbb3Vbl5WVZfXq1avW+/Q1X9R4tsOHD1uSrA0bNtRNp73gq/qOHz9utW/f3lq3bp118803N9jJxhf1/eY3v7Fuuukm33S4FnxR41133WX98pe/dGszaNAga+jQoXXYc894W1/Lli2tmTNnuq07u+/+fp7xpMaz+dN5xtP6/PU840l99XGe4fbTD5w6dUr5+flKSUlxrQsMDFRKSoo2bdpU7TY33nij8vPzXZflCgoK9MYbb6hfv3613qcv+aLG6pSWlkqSmjZtWoe9vzBf1peRkaG77rrLbd/1zVf1vfbaa+rWrZt+9rOfKSYmRtdff71eeOEF3xZzHr6q8cYbb1ReXp4+/fRTSdK//vUvbdy4UXfeeacPqzlXbeqrqKiQw+FwWxcaGqqNGzfWep++5Isaq+NP5xlP6/PX84wn9dXHeeaSeaGlJ44ePaqqqirFxsa6rY+NjdWePXuq3eb+++/X0aNHddNNN8myLJ0+fVqjRo3Sb3/721rv05d8UePZnE6nxowZo169eqlTp051XkNNfFXf0qVLtX37dn3wwQc+7f+F+Kq+goICzZkzR1lZWfrtb3+rDz74QI8++qiCg4OVlpbm05rO5qsax40bp7KyMnXs2FFBQUGqqqrSU089paFDh/q0nrPVpr7U1FRNnTpVvXv3VmJiovLy8rRy5UpVVVXVep++5Isaz+Zv5xlP6vPn84wn9dXHeYYrNRdp/fr1evrppzV79mxt375dK1eu1Ouvv64pU6Y0dNfqjLc1ZmRkaNeuXVq6dGk997R2LlTfwYMH9dhjj2nRokXn/CXiDzz5/TmdTv3Xf/2Xnn76aV1//fUaOXKkRowYoeeee64Be+45T2p85ZVXtGjRIi1evFjbt2/XggUL9Oc//1kLFixowJ57Zvr06Wrfvr06duyo4OBgZWZmKj09XYGB5pzCva3R384zF6rP388znvz+6uU849ObW36moqLCCgoKslatWuW2ftiwYdY999xT7TY33XST9cQTT7itW7hwoRUaGmpVVVXVap++5IsafygjI8Nq1aqVVVBQUKf99pQv6lu1apUlyQoKCnItkqyAgAArKCjIOn36tK/KOYevfn+tW7e2HnroIbc2s2fPtuLi4uqu8x7yVY2tWrU6557/lClTrA4dOtRd5z1wMeeE7777zvryyy8tp9NpjR071rrmmmsuep++4Isaf8gfzzNnnK8+fz/PnFHT768+zjPmxPw6EBwcrK5duyovL8+1zul0Ki8vTz179qx2mxMnTpzzl0RQUJAkybKsWu3Tl3xR45mvmZmZWrVqld555x21bdvWRxXUzBf13Xbbbfroo4+0c+dO19KtWzcNHTpUO3fudLWtD776/fXq1eucR2M//fRTXXnllXXZfY/4qsbztXE6nXXZ/Qu6mHOCw+FQfHy8Tp8+rRUrVqh///4XvU9f8EWNkn+fZ844X33+fp45o6bfX72cZ+osHhli6dKlVkhIiPXiiy9au3fvtkaOHGlFRUVZxcXFlmVZ1oMPPmiNGzfO1X7ixIlWeHi4tWTJEqugoMD6v//7PysxMdH6+c9/7vE+65svahw9erQVGRlprV+/3ioqKnItJ06cMKK+szXkUwm+qG/r1q1WkyZNrKeeesrat2+ftWjRIuvyyy+3Xn755Xqvz7J8U2NaWpoVHx/veqR75cqVVnR0tDV27NhGX9/mzZutFStWWJ9//rn13nvvWbfeeqvVtm1b65tvvvF4n/XNFzX683nGk/rO5k/nGU/qq4/zDKGmGjNmzLBat25tBQcHWz169LA2b97s+tnNN99spaWlub6vrKy0Jk2aZCUmJloOh8NKSEiwHn744XP+R61pnw2hrmuUVO0yf/78+ivqB3zxO/yhhjzZWJZv6luzZo3VqVMnKyQkxOrYsaP1/PPP11M11avrGsvKyqzHHnvMat26teVwOKx27dpZTz75pFVRUVGPVX3Pm/rWr19vXX311VZISIjVrFkz68EHH7QKCwu92mdDqOsa/fk84+nv8If86TzjaX2+Ps8EWFYDTacJAABQhxhTAwAAjECoAQAARiDUAAAAIxBqAACAEQg1AADACIQaAABgBEINAAAwAqEGAAAYgVADAACMQKgBAABGINQAAAAjEGoAAIAR/j904zH5ctyjNAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "df=data\n",
        "df=df.sort_values(by='conf',ascending=False,ignore_index=True)\n",
        "# Assuming your dataframe is called 'df'\n",
        "total_rows = df.shape[0]\n",
        "num_rows_20_percent = int(total_rows * 1) #can change this to 0.2, 0.4, 0.6, 0.8\n",
        "\n",
        "\n",
        "df['allocate'] = 0  # Initialize the new column with 0\n",
        "df.loc[:num_rows_20_percent-1, 'allocate'] = 1  # Assign 1 to the first 20% of rows\n",
        "\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "IPIz9pYKjgM4",
        "outputId": "71c9a5bd-3dae-4141-bca8-acfbd101f9ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  Class Index      conf  \\\n",
              "0  Title: Cassini finds new Saturn moons Descript...            4  1.000000   \n",
              "1  Title: Thorpe Wins Men's 200 Freestyle Descrip...            2  0.994444   \n",
              "2  Title: What are the best cities for business i...            3  0.994444   \n",
              "3  Title: Editor's Picks: The Best of our Web Ser...            4  0.994444   \n",
              "4  Title: Google seeks to end auction Tuesday Des...            4  0.988889   \n",
              "\n",
              "   MV_world  MV_sports  MV_business  MV_sci  allocate  \n",
              "0         0          0            0       1         1  \n",
              "1         0          1            0       0         1  \n",
              "2         0          0            1       0         1  \n",
              "3         0          0            0       1         1  \n",
              "4         0          0            1       0         1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-dbeb277c-460a-45c3-9d83-f0d6b498a743\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Class Index</th>\n",
              "      <th>conf</th>\n",
              "      <th>MV_world</th>\n",
              "      <th>MV_sports</th>\n",
              "      <th>MV_business</th>\n",
              "      <th>MV_sci</th>\n",
              "      <th>allocate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Title: Cassini finds new Saturn moons Descript...</td>\n",
              "      <td>4</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Title: Thorpe Wins Men's 200 Freestyle Descrip...</td>\n",
              "      <td>2</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Title: What are the best cities for business i...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Title: Editor's Picks: The Best of our Web Ser...</td>\n",
              "      <td>4</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Title: Google seeks to end auction Tuesday Des...</td>\n",
              "      <td>4</td>\n",
              "      <td>0.988889</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dbeb277c-460a-45c3-9d83-f0d6b498a743')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-dbeb277c-460a-45c3-9d83-f0d6b498a743 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-dbeb277c-460a-45c3-9d83-f0d6b498a743');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 265
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined\n",
        "df['gold_label']=df['Class Index']\n",
        "world=[]\n",
        "sports=[]\n",
        "business=[]\n",
        "sci=[]\n",
        "chatgpt_count=0\n",
        "for index, row in df.iterrows():\n",
        "    if row['allocate']==1:\n",
        "      world.append(row['MV_world'])\n",
        "      sports.append(row['MV_sports'])\n",
        "      business.append(row['MV_business'])\n",
        "      sci.append(row['MV_sci'])\n",
        "    else:\n",
        "      if row['gold_label']==1:\n",
        "        world.append(1)\n",
        "        sports.append(0)\n",
        "        business.append(0)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==2:\n",
        "        world.append(0)\n",
        "        sports.append(1)\n",
        "        business.append(0)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==3:\n",
        "        world.append(0)\n",
        "        sports.append(0)\n",
        "        business.append(1)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==4:\n",
        "        world.append(0)\n",
        "        sports.append(0)\n",
        "        business.append(0)\n",
        "        sci.append(1)\n",
        "df['world']=world\n",
        "df['sports']=sports\n",
        "df['business']=business\n",
        "df['sci']=sci\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "id": "WxS4YXfA730N",
        "outputId": "a64947e7-9c5c-4775-9cdb-583257d67e8a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  Class Index      conf  \\\n",
              "0  Title: Cassini finds new Saturn moons Descript...            4  1.000000   \n",
              "1  Title: Thorpe Wins Men's 200 Freestyle Descrip...            2  0.994444   \n",
              "2  Title: What are the best cities for business i...            3  0.994444   \n",
              "3  Title: Editor's Picks: The Best of our Web Ser...            4  0.994444   \n",
              "4  Title: Google seeks to end auction Tuesday Des...            4  0.988889   \n",
              "\n",
              "   MV_world  MV_sports  MV_business  MV_sci  allocate  gold_label  world  \\\n",
              "0         0          0            0       1         1           4      0   \n",
              "1         0          1            0       0         1           2      0   \n",
              "2         0          0            1       0         1           3      0   \n",
              "3         0          0            0       1         1           4      0   \n",
              "4         0          0            1       0         1           4      0   \n",
              "\n",
              "   sports  business  sci  \n",
              "0       0         0    1  \n",
              "1       1         0    0  \n",
              "2       0         1    0  \n",
              "3       0         0    1  \n",
              "4       0         1    0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e2dfcfea-0e3f-4163-b87f-a982f83839d4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Class Index</th>\n",
              "      <th>conf</th>\n",
              "      <th>MV_world</th>\n",
              "      <th>MV_sports</th>\n",
              "      <th>MV_business</th>\n",
              "      <th>MV_sci</th>\n",
              "      <th>allocate</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Title: Cassini finds new Saturn moons Descript...</td>\n",
              "      <td>4</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Title: Thorpe Wins Men's 200 Freestyle Descrip...</td>\n",
              "      <td>2</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Title: What are the best cities for business i...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Title: Editor's Picks: The Best of our Web Ser...</td>\n",
              "      <td>4</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Title: Google seeks to end auction Tuesday Des...</td>\n",
              "      <td>4</td>\n",
              "      <td>0.988889</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e2dfcfea-0e3f-4163-b87f-a982f83839d4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e2dfcfea-0e3f-4163-b87f-a982f83839d4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e2dfcfea-0e3f-4163-b87f-a982f83839d4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 266
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=df.drop(91)"
      ],
      "metadata": {
        "id": "U3lTa8eaG_fJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df=df.drop(484)"
      ],
      "metadata": {
        "id": "1nC2JvvutU0r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df=df.drop(1116)"
      ],
      "metadata": {
        "id": "_iUDQlPsvdS6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dLcetMjZFjSH"
      },
      "source": [
        "# Fine-tuning\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dal0ggBcYdD"
      },
      "source": [
        "Dataset will be tokenized then split into training and validation sets. The validation set will be used to monitor training. For testing a separate test set will be loaded for analysis."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0ecREc7GnISW",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "#df = data #jigsaw-toxic-comment-classification-challenge"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UVI59S9VaAfB",
        "outputId": "bfed3bd9-a201-49f4-83b9-a05166854d8b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Label columns:  ['world', 'sports', 'business', 'sci']\n"
          ]
        }
      ],
      "source": [
        "#combined\n",
        "cols = df.columns\n",
        "label_cols = list(cols[9:13])\n",
        "num_labels = len(label_cols)\n",
        "print('Label columns: ', label_cols)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xzgA5qQgYIBZ",
        "outputId": "19baa00f-432c-4794-dc13-af4698b63692"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Count of 1 per label: \n",
            " world       270\n",
            "sports      272\n",
            "business    418\n",
            "sci         202\n",
            "dtype: int64 \n",
            "\n",
            "Count of 0 per label: \n",
            " world       881\n",
            "sports      879\n",
            "business    733\n",
            "sci         949\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "print('Count of 1 per label: \\n', df[label_cols].sum(), '\\n') # Label counts, may need to downsample or upsample\n",
        "print('Count of 0 per label: \\n', df[label_cols].eq(0).sum())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 354
        },
        "id": "0DF3ddjej5vd",
        "outputId": "9fe124a7-711f-4229-c788-658fbe8a592c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  Class Index      conf  \\\n",
              "0  Title: Cassini finds new Saturn moons Descript...            4  1.000000   \n",
              "1  Title: Thorpe Wins Men's 200 Freestyle Descrip...            2  0.994444   \n",
              "2  Title: What are the best cities for business i...            3  0.994444   \n",
              "3  Title: Editor's Picks: The Best of our Web Ser...            4  0.994444   \n",
              "4  Title: Google seeks to end auction Tuesday Des...            4  0.988889   \n",
              "\n",
              "   MV_world  MV_sports  MV_business  MV_sci  allocate  gold_label  world  \\\n",
              "0         0          0            0       1         1           4      0   \n",
              "1         0          1            0       0         1           2      0   \n",
              "2         0          0            1       0         1           3      0   \n",
              "3         0          0            0       1         1           4      0   \n",
              "4         0          0            1       0         1           4      0   \n",
              "\n",
              "   sports  business  sci one_hot_labels  \n",
              "0       0         0    1   [0, 0, 0, 1]  \n",
              "1       1         0    0   [0, 1, 0, 0]  \n",
              "2       0         1    0   [0, 0, 1, 0]  \n",
              "3       0         0    1   [0, 0, 0, 1]  \n",
              "4       0         1    0   [0, 0, 1, 0]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a7513293-a084-4886-92b7-387e3e96dba1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Class Index</th>\n",
              "      <th>conf</th>\n",
              "      <th>MV_world</th>\n",
              "      <th>MV_sports</th>\n",
              "      <th>MV_business</th>\n",
              "      <th>MV_sci</th>\n",
              "      <th>allocate</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Title: Cassini finds new Saturn moons Descript...</td>\n",
              "      <td>4</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Title: Thorpe Wins Men's 200 Freestyle Descrip...</td>\n",
              "      <td>2</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 1, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Title: What are the best cities for business i...</td>\n",
              "      <td>3</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Title: Editor's Picks: The Best of our Web Ser...</td>\n",
              "      <td>4</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Title: Google seeks to end auction Tuesday Des...</td>\n",
              "      <td>4</td>\n",
              "      <td>0.988889</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a7513293-a084-4886-92b7-387e3e96dba1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a7513293-a084-4886-92b7-387e3e96dba1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a7513293-a084-4886-92b7-387e3e96dba1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 279
        }
      ],
      "source": [
        "df['one_hot_labels'] = list(df[label_cols].values)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MlhHifh5bW7e"
      },
      "outputs": [],
      "source": [
        "labels = list(df.one_hot_labels.values)\n",
        "comments = list(df.text.values)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlMHfElhGJzc"
      },
      "source": [
        "Load the pretrained tokenizer that corresponds to your choice in model. e.g.,\n",
        "\n",
        "```\n",
        "BERT:\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "\n",
        "XLNet:\n",
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=False)\n",
        "\n",
        "RoBERTa:\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False)\n",
        "```\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhVr8SziL_PY"
      },
      "source": [
        "In order to avoid memory issues with Google Colab, I enforce a max_length of 100 tokens. Note that some sentences may not adequately represent each label because of this."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HNsEu-vUur-4",
        "outputId": "330ca567-5ece-48ee-d873-86b4a1804f9e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tokenizer outputs:  dict_keys(['input_ids', 'attention_mask'])\n"
          ]
        }
      ],
      "source": [
        "from transformers import RobertaTokenizer\n",
        "max_length = 100\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False) # tokenizer\n",
        "encodings = tokenizer.batch_encode_plus(comments,max_length=max_length,pad_to_max_length=True) # tokenizer's encoding method\n",
        "print('tokenizer outputs: ', encodings.keys())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nG9vOTp92yZE",
        "outputId": "2c83c9c3-acd6-4cee-eda5-d9a52d355873"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'attention_mask'])"
            ]
          },
          "metadata": {},
          "execution_count": 118
        }
      ],
      "source": [
        "encodings.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l6CCLSjfur-9"
      },
      "outputs": [],
      "source": [
        "input_ids = encodings['input_ids'] # tokenized and encoded sentences\n",
        "#token_type_ids = encodings['token_type_ids'] # token type ids\n",
        "attention_masks = encodings['attention_mask'] # attention masks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vSOFbThlYcpb",
        "outputId": "351dc3dd-2b9e-465d-b5db-b04b5ae39cd7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "df label indices with only one instance:  []\n"
          ]
        }
      ],
      "source": [
        "# Identifying indices of 'one_hot_labels' entries that only occur once - this will allow us to stratify split our training data later\n",
        "label_counts = df.one_hot_labels.astype(str).value_counts()\n",
        "one_freq = label_counts[label_counts==1].keys()\n",
        "one_freq_idxs = sorted(list(df[df.one_hot_labels.astype(str).isin(one_freq)].index), reverse=True)\n",
        "print('df label indices with only one instance: ', one_freq_idxs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CQQ7CoOag_r7"
      },
      "outputs": [],
      "source": [
        "# Gathering single instance inputs to force into the training set after stratified split\n",
        "one_freq_input_ids = [input_ids.pop(i) for i in one_freq_idxs]\n",
        "#one_freq_token_types = [token_type_ids.pop(i) for i in one_freq_idxs]\n",
        "one_freq_attention_masks = [attention_masks.pop(i) for i in one_freq_idxs]\n",
        "one_freq_labels = [labels.pop(i) for i in one_freq_idxs]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9PxAt48HRRj"
      },
      "source": [
        "Be sure to handle all classes during validation using \"stratify\" during train/validation split:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WPFaq4ufnIT2"
      },
      "outputs": [],
      "source": [
        "# Use train_test_split to split our data into train and validation sets\n",
        "\n",
        "train_inputs, validation_inputs, train_labels, validation_labels, train_masks, validation_masks = train_test_split(input_ids, labels, attention_masks,\n",
        "                                                            random_state=20, test_size=0.10, stratify = labels)\n",
        "\n",
        "# Add one frequency data to train data\n",
        "train_inputs.extend(one_freq_input_ids)\n",
        "train_labels.extend(one_freq_labels)\n",
        "train_masks.extend(one_freq_attention_masks)\n",
        "#train_token_types.extend(one_freq_token_types)\n",
        "\n",
        "# Convert all of our data into torch tensors, the required datatype for our model\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "#train_token_types = torch.tensor(train_token_types)\n",
        "\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "validation_masks = torch.tensor(validation_masks)\n",
        "#validation_token_types = torch.tensor(validation_token_types)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZRnuLna-nIT4"
      },
      "outputs": [],
      "source": [
        "# Select a batch size for training. For fine-tuning with XLNet, the authors recommend a batch size of 32, 48, or 128. We will use 32 here to avoid memory issues.\n",
        "batch_size = 16\n",
        "\n",
        "# Create an iterator of our data with torch DataLoader. This helps save on memory during training because, unlike a for loop,\n",
        "# with an iterator the entire dataset does not need to be loaded into memory\n",
        "\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iiFRnP_ZTBFa"
      },
      "outputs": [],
      "source": [
        "torch.save(validation_dataloader,'validation_data_loader')\n",
        "torch.save(train_dataloader,'train_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncGteBuSFuZM"
      },
      "source": [
        "## Load Model & Set Params"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0dL-Bz_NrGj"
      },
      "source": [
        "Load the appropriate model below, each model already contains a single dense layer for classification on top.\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "BERT:\n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=num_labels)\n",
        "\n",
        "XLNet:\n",
        "model = XLNetForSequenceClassification.from_pretrained(\"xlnet-base-cased\", num_labels=num_labels)\n",
        "\n",
        "RoBERTa:\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ujk4k16DnIT6",
        "outputId": "9d459480-6e99-4499-e2b2-886b7f0a2693"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=4, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 288
        }
      ],
      "source": [
        "# Load model, the pretrained model will include a single linear classification layer on top for classification.\n",
        "from transformers import RobertaForSequenceClassification\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "model.cuda()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGE4gv9qfhRG"
      },
      "source": [
        "Setting custom optimization parameters for the AdamW optimizer https://huggingface.co/transformers/main_classes/optimizer_schedules.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GsV8zwWYnIT9"
      },
      "outputs": [],
      "source": [
        "# setting custom optimization parameters. You may implement a scheduler here as well.\n",
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'gamma', 'beta']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aOomZIEIoHOL",
        "outputId": "6b82e56f-553c-4bc5-fce7-b7891f42bac5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import AdamW\n",
        "optimizer = AdamW(optimizer_grouped_parameters,lr=2e-5 ,correct_bias=True)\n",
        "# optimizer = AdamW(model.parameters(),lr=2e-5)  # Default optimization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JRQQZ8zIFzLW"
      },
      "source": [
        "## Train Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ho6xx8cQfhvK"
      },
      "outputs": [],
      "source": [
        "def is_max(array):\n",
        "  max_val=max(array)\n",
        "  res=[i==max_val for i in array]\n",
        "  return res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        },
        "id": "uDLZmEC_oKo3",
        "outputId": "59160ef7-5810-43cd-a4b8-ea9eb4790dc0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch:   0%|          | 0/10 [00:00<?, ?it/s]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "OutOfMemoryError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-291-88a02eb84f47>\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0;31m# Forward pass for multilabel classification\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb_input_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mb_input_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m     \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0mloss_func\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBCEWithLogitsLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/roberta/modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1214\u001b[0m         \u001b[0mreturn_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_return_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1216\u001b[0;31m         outputs = self.roberta(\n\u001b[0m\u001b[1;32m   1217\u001b[0m             \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1218\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/roberta/modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    850\u001b[0m             \u001b[0mpast_key_values_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpast_key_values_length\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    851\u001b[0m         )\n\u001b[0;32m--> 852\u001b[0;31m         encoder_outputs = self.encoder(\n\u001b[0m\u001b[1;32m    853\u001b[0m             \u001b[0membedding_output\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mextended_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/roberta/modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    525\u001b[0m                 )\n\u001b[1;32m    526\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 527\u001b[0;31m                 layer_outputs = layer_module(\n\u001b[0m\u001b[1;32m    528\u001b[0m                     \u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m                     \u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/roberta/modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    409\u001b[0m         \u001b[0;31m# decoder uni-directional self-attention cached key/values tuple is at positions 1,2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    410\u001b[0m         \u001b[0mself_attn_past_key_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpast_key_value\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpast_key_value\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 411\u001b[0;31m         self_attention_outputs = self.attention(\n\u001b[0m\u001b[1;32m    412\u001b[0m             \u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    413\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/roberta/modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    336\u001b[0m         \u001b[0moutput_attentions\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    337\u001b[0m     ) -> Tuple[torch.Tensor]:\n\u001b[0;32m--> 338\u001b[0;31m         self_outputs = self.self(\n\u001b[0m\u001b[1;32m    339\u001b[0m             \u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/roberta/modeling_roberta.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    232\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0;31m# Take the dot product between \"query\" and \"key\" to get the raw attention scores.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m         \u001b[0mattention_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery_layer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey_layer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"relative_key\"\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mposition_embedding_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"relative_key_query\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 14.75 GiB total capacity; 12.88 GiB already allocated; 12.81 MiB free; 13.68 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
          ]
        }
      ],
      "source": [
        "# Store our loss and accuracy for plotting\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 10\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "\n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss()\n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss()\n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())\n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  #threshold = 0.30\n",
        "  #pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  #print(pred_labels)\n",
        "  #max_val = max(pred_labels)\n",
        "  pred_bools = [is_max(arr) for arr in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Store our loss and accuracy for plotting\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 5\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "\n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss()\n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss()\n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())\n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  #threshold = 0.30\n",
        "  #pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  #print(pred_labels)\n",
        "  #max_val = max(pred_labels)\n",
        "  pred_bools = [is_max(arr) for arr in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aZyAZ7eUC9Du",
        "outputId": "8e3b9815-50a8-4b67-d5b5-568d29d53121"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/5 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.025419917917595458\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  20%|██        | 1/5 [00:18<01:14, 18.53s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  82.75862068965517\n",
            "Flat Validation Accuracy:  82.75862068965517\n",
            "Train loss: 0.021188997162076142\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  40%|████      | 2/5 [00:37<00:56, 18.79s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  81.03448275862068\n",
            "Flat Validation Accuracy:  81.03448275862068\n",
            "Train loss: 0.021381761334263363\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  60%|██████    | 3/5 [00:56<00:37, 18.79s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  81.89655172413794\n",
            "Flat Validation Accuracy:  81.89655172413794\n",
            "Train loss: 0.02045951306533355\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  80%|████████  | 4/5 [01:14<00:18, 18.67s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  83.62068965517241\n",
            "Flat Validation Accuracy:  83.62068965517241\n",
            "Train loss: 0.01706601111218333\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 5/5 [01:33<00:00, 18.65s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  81.89655172413794\n",
            "Flat Validation Accuracy:  81.89655172413794\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluation"
      ],
      "metadata": {
        "id": "l6_zh67P9NF2"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_7dd2GE3F4yK"
      },
      "source": [
        "## Load and Preprocess Test Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        },
        "id": "G5Q7hC4GFOLJ",
        "outputId": "b7a7cf0a-c1ff-47b2-f2b1-46c3a94d0ec8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     Class Index                                         Title  \\\n",
              "495            2                               Pressure points   \n",
              "496            3  Unions protest as overtime rules take effect   \n",
              "497            1              Serb denies siege terror charges   \n",
              "498            2                 11th-hour highlights too late   \n",
              "499            4                             No IE? No Can See   \n",
              "\n",
              "                                           Description  \\\n",
              "495  ATHENS -- The booing went on for nearly 10 min...   \n",
              "496  WASHINGTON -- Hundreds of workers rallied on t...   \n",
              "497  A Bosnian Serb general accused of organising t...   \n",
              "498  NBC's prime-time Olympic coverage is taped and...   \n",
              "499  One thing that #39;s always irritated those wh...   \n",
              "\n",
              "                                                  text  gold_label  world  \\\n",
              "495  Title: Pressure points Description: ATHENS -- ...           2      0   \n",
              "496  Title: Unions protest as overtime rules take e...           3      0   \n",
              "497  Title: Serb denies siege terror charges Descri...           1      1   \n",
              "498  Title: 11th-hour highlights too late Descripti...           2      0   \n",
              "499  Title: No IE? No Can See Description: One thin...           4      0   \n",
              "\n",
              "     sports  business  sci  \n",
              "495       1         0    0  \n",
              "496       0         1    0  \n",
              "497       0         0    0  \n",
              "498       1         0    0  \n",
              "499       0         0    1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-13f3d45e-96cb-48c3-952a-653ddf8b73e3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>text</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>495</th>\n",
              "      <td>2</td>\n",
              "      <td>Pressure points</td>\n",
              "      <td>ATHENS -- The booing went on for nearly 10 min...</td>\n",
              "      <td>Title: Pressure points Description: ATHENS -- ...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>496</th>\n",
              "      <td>3</td>\n",
              "      <td>Unions protest as overtime rules take effect</td>\n",
              "      <td>WASHINGTON -- Hundreds of workers rallied on t...</td>\n",
              "      <td>Title: Unions protest as overtime rules take e...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>497</th>\n",
              "      <td>1</td>\n",
              "      <td>Serb denies siege terror charges</td>\n",
              "      <td>A Bosnian Serb general accused of organising t...</td>\n",
              "      <td>Title: Serb denies siege terror charges Descri...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>498</th>\n",
              "      <td>2</td>\n",
              "      <td>11th-hour highlights too late</td>\n",
              "      <td>NBC's prime-time Olympic coverage is taped and...</td>\n",
              "      <td>Title: 11th-hour highlights too late Descripti...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499</th>\n",
              "      <td>4</td>\n",
              "      <td>No IE? No Can See</td>\n",
              "      <td>One thing that #39;s always irritated those wh...</td>\n",
              "      <td>Title: No IE? No Can See Description: One thin...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-13f3d45e-96cb-48c3-952a-653ddf8b73e3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-13f3d45e-96cb-48c3-952a-653ddf8b73e3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-13f3d45e-96cb-48c3-952a-653ddf8b73e3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "test_df = pd.read_csv('agnews_test.csv')\n",
        "#test_labels_df = pd.read_csv('test_labels.csv')\n",
        "#test_df = test_df.merge(test_labels_df, on='id', how='left')\n",
        "# test_df['text']=test_df['sentence']\n",
        "test_df['world']=test_df['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
        "test_df['sports']=test_df['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
        "test_df['business']=test_df['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
        "test_df['sci']=test_df['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
        "#print('Null values: ', test_df.isnull().values.any()) #should not be any null sentences or labels\n",
        "#print('Same columns between train and test: ', label_cols == test_label_cols) #columns should be the same\n",
        "test_df.tail()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j_PZ1mVOhkK8",
        "outputId": "e099c49f-ead0-44fb-def7-94b4bad00cd9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['world', 'sports', 'business', 'sci']"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "test_label_cols = list(test_df.columns[5:])\n",
        "test_label_cols"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "id": "77rjCrMGpYxz",
        "outputId": "22dfffb7-6a6b-49e7-90a7-1c160d0393bb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                              Title  \\\n",
              "0            3                  Fears for T N pension after talks   \n",
              "1            4  The Race is On: Second Private Team Sets Launc...   \n",
              "2            4      Ky. Company Wins Grant to Study Peptides (AP)   \n",
              "3            4      Prediction Unit Helps Forecast Wildfires (AP)   \n",
              "4            4        Calif. Aims to Limit Farm-Related Smog (AP)   \n",
              "\n",
              "                                         Description  \\\n",
              "0  Unions representing workers at Turner   Newall...   \n",
              "1  SPACE.com - TORONTO, Canada -- A second\\team o...   \n",
              "2  AP - A company founded by a chemistry research...   \n",
              "3  AP - It's barely dawn when Mike Fitzpatrick st...   \n",
              "4  AP - Southern California's smog-fighting agenc...   \n",
              "\n",
              "                                                text  gold_label  world  \\\n",
              "0  Title: Fears for T N pension after talks Descr...           3      0   \n",
              "1  Title: The Race is On: Second Private Team Set...           4      0   \n",
              "2  Title: Ky. Company Wins Grant to Study Peptide...           4      0   \n",
              "3  Title: Prediction Unit Helps Forecast Wildfire...           4      0   \n",
              "4  Title: Calif. Aims to Limit Farm-Related Smog ...           4      0   \n",
              "\n",
              "   sports  business  sci one_hot_labels  \n",
              "0       0         1    0   [0, 0, 1, 0]  \n",
              "1       0         0    1   [0, 0, 0, 1]  \n",
              "2       0         0    1   [0, 0, 0, 1]  \n",
              "3       0         0    1   [0, 0, 0, 1]  \n",
              "4       0         0    1   [0, 0, 0, 1]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-be99eef2-1e1c-48c0-ba7f-1d449b95290e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>text</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Fears for T N pension after talks</td>\n",
              "      <td>Unions representing workers at Turner   Newall...</td>\n",
              "      <td>Title: Fears for T N pension after talks Descr...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>The Race is On: Second Private Team Sets Launc...</td>\n",
              "      <td>SPACE.com - TORONTO, Canada -- A second\\team o...</td>\n",
              "      <td>Title: The Race is On: Second Private Team Set...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>Ky. Company Wins Grant to Study Peptides (AP)</td>\n",
              "      <td>AP - A company founded by a chemistry research...</td>\n",
              "      <td>Title: Ky. Company Wins Grant to Study Peptide...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>Prediction Unit Helps Forecast Wildfires (AP)</td>\n",
              "      <td>AP - It's barely dawn when Mike Fitzpatrick st...</td>\n",
              "      <td>Title: Prediction Unit Helps Forecast Wildfire...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>Calif. Aims to Limit Farm-Related Smog (AP)</td>\n",
              "      <td>AP - Southern California's smog-fighting agenc...</td>\n",
              "      <td>Title: Calif. Aims to Limit Farm-Related Smog ...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-be99eef2-1e1c-48c0-ba7f-1d449b95290e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-be99eef2-1e1c-48c0-ba7f-1d449b95290e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-be99eef2-1e1c-48c0-ba7f-1d449b95290e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "test_df = test_df[~test_df[test_label_cols].eq(-1).any(axis=1)] #remove irrelevant rows/comments with -1 values\n",
        "test_df['one_hot_labels'] = list(test_df[test_label_cols].values)\n",
        "test_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1a41OmU2i7qp"
      },
      "outputs": [],
      "source": [
        "# Gathering input data\n",
        "test_labels = list(test_df.one_hot_labels.values)\n",
        "test_comments = list(test_df.text.values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "amySMO8EQzf2",
        "outputId": "640569a5-e0f4-4b77-96e8-d2b5a001d887"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Encoding input data\n",
        "test_encodings = tokenizer.batch_encode_plus(test_comments,max_length=max_length,pad_to_max_length=True)\n",
        "test_input_ids = test_encodings['input_ids']\n",
        "#test_token_type_ids = test_encodings['token_type_ids']\n",
        "test_attention_masks = test_encodings['attention_mask']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hqOfi9fkRaRN"
      },
      "outputs": [],
      "source": [
        "# Make tensors out of data\n",
        "test_inputs = torch.tensor(test_input_ids)\n",
        "test_labels = torch.tensor(test_labels)\n",
        "test_masks = torch.tensor(test_attention_masks)\n",
        "#test_token_types = torch.tensor(test_token_type_ids)\n",
        "# Create test dataloader\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "#test_data = TensorDataset(test_inputs, test_masks)\n",
        "test_sampler = SequentialSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)\n",
        "# Save test dataloader\n",
        "torch.save(test_dataloader,'test_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PFTWxCA_GBau"
      },
      "source": [
        "## Prediction and Metics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NPvrL6OFSQvf"
      },
      "outputs": [],
      "source": [
        "# Test\n",
        "\n",
        "# Put model in evaluation mode to evaluate loss on the validation set\n",
        "model.eval()\n",
        "\n",
        "#track variables\n",
        "logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "# Predict\n",
        "for i, batch in enumerate(test_dataloader):\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  #b_input_ids, b_input_mask = batch\n",
        "  with torch.no_grad():\n",
        "    # Forward pass\n",
        "    outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    b_logit_pred = outs[0]\n",
        "    pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "    b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "    pred_label = pred_label.to('cpu').numpy()\n",
        "    b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "  tokenized_texts.append(b_input_ids)\n",
        "  logit_preds.append(b_logit_pred)\n",
        "  true_labels.append(b_labels)\n",
        "  pred_labels.append(pred_label)\n",
        "\n",
        "# Flatten outputs\n",
        "tokenized_texts = [item for sublist in tokenized_texts for item in sublist]\n",
        "pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "true_labels = [item for sublist in true_labels for item in sublist]\n",
        "# Converting flattened binary values to boolean values\n",
        "true_bools = [tl==1 for tl in true_labels]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bQeGWqeMzAoZ"
      },
      "source": [
        "We need to threshold our sigmoid function outputs which range from [0, 1]. Below I use 0.50 as a threshold."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BZcZUcYOxxmM",
        "outputId": "3bc80000-b883-4485-ab80-012fb035e16f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test F1 Accuracy:  0.88\n",
            "Test Flat Accuracy:  0.88 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.85      0.92      0.88       122\n",
            "      sports       0.96      0.94      0.95       145\n",
            "    business       0.81      0.76      0.79       106\n",
            "         sci       0.88      0.87      0.87       127\n",
            "\n",
            "   micro avg       0.88      0.88      0.88       500\n",
            "   macro avg       0.87      0.87      0.87       500\n",
            "weighted avg       0.88      0.88      0.88       500\n",
            " samples avg       0.88      0.88      0.88       500\n",
            "\n"
          ]
        }
      ],
      "source": [
        "#all human\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.2\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XtvNKv6FlOWi",
        "outputId": "0fede114-e086-4908-9939-3123695f9e10"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.860164065271578\n",
            "Micro Test F1 Accuracy:  0.868\n",
            "Weighted Test F1 Accuracy:  0.867295200762876\n",
            "Test Flat Accuracy:  0.868 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.86      0.89      0.87       122\n",
            "      sports       0.95      0.98      0.96       145\n",
            "    business       0.77      0.78      0.78       106\n",
            "         sci       0.87      0.80      0.83       127\n",
            "\n",
            "   micro avg       0.87      0.87      0.87       500\n",
            "   macro avg       0.86      0.86      0.86       500\n",
            "weighted avg       0.87      0.87      0.87       500\n",
            " samples avg       0.87      0.87      0.87       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.4\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NNVTY2mksKGO",
        "outputId": "ebefccc4-095d-42c9-8890-5bbf5e91935d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8489270374996141\n",
            "Micro Test F1 Accuracy:  0.8599999999999999\n",
            "Weighted Test F1 Accuracy:  0.8574014009534355\n",
            "Test Flat Accuracy:  0.86 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.87      0.88      0.87       122\n",
            "      sports       0.95      0.94      0.95       145\n",
            "    business       0.82      0.65      0.73       106\n",
            "         sci       0.79      0.92      0.85       127\n",
            "\n",
            "   micro avg       0.86      0.86      0.86       500\n",
            "   macro avg       0.86      0.85      0.85       500\n",
            "weighted avg       0.86      0.86      0.86       500\n",
            " samples avg       0.86      0.86      0.86       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.6\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K71A8Tkedoxu",
        "outputId": "8f24a004-1a78-47fb-95aa-e565d4e01cc2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8409243667370322\n",
            "Micro Test F1 Accuracy:  0.848\n",
            "Weighted Test F1 Accuracy:  0.8488411074868725\n",
            "Test Flat Accuracy:  0.848 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.91      0.85      0.88       122\n",
            "      sports       0.95      0.98      0.97       145\n",
            "    business       0.64      0.91      0.75       106\n",
            "         sci       0.94      0.65      0.77       127\n",
            "\n",
            "   micro avg       0.85      0.85      0.85       500\n",
            "   macro avg       0.86      0.85      0.84       500\n",
            "weighted avg       0.87      0.85      0.85       500\n",
            " samples avg       0.85      0.85      0.85       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.8\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zJlvIxIntlK_",
        "outputId": "58ab7680-9f69-4058-aa60-0e24697e5ff0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8214350559112206\n",
            "Micro Test F1 Accuracy:  0.83\n",
            "Weighted Test F1 Accuracy:  0.8300533363440571\n",
            "Test Flat Accuracy:  0.83 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.89      0.84      0.86       122\n",
            "      sports       0.95      0.98      0.96       145\n",
            "    business       0.61      0.90      0.73       106\n",
            "         sci       0.95      0.60      0.73       127\n",
            "\n",
            "   micro avg       0.83      0.83      0.83       500\n",
            "   macro avg       0.85      0.83      0.82       500\n",
            "weighted avg       0.86      0.83      0.83       500\n",
            " samples avg       0.83      0.83      0.83       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 1\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cyl_lFs4vscp",
        "outputId": "d77b3715-1593-4ee3-8201-10d8929741fe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8214350559112206\n",
            "Micro Test F1 Accuracy:  0.83\n",
            "Weighted Test F1 Accuracy:  0.8300533363440571\n",
            "Test Flat Accuracy:  0.83 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.89      0.84      0.86       122\n",
            "      sports       0.95      0.98      0.96       145\n",
            "    business       0.61      0.90      0.73       106\n",
            "         sci       0.95      0.60      0.73       127\n",
            "\n",
            "   micro avg       0.83      0.83      0.83       500\n",
            "   macro avg       0.85      0.83      0.82       500\n",
            "weighted avg       0.86      0.83      0.83       500\n",
            " samples avg       0.83      0.83      0.83       500\n",
            "\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}