EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "awTaxwYVO3MB"
   },
   "source": [
    "# EDA for COWLS2h Dataset\n",
    "This notebook covers basic EDA on the COWS-L2H dataset. It also includes feature engineering of evidence words using Chat-GPT."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n43cbXdNO3MD",
    "tags": []
   },
   "source": [
    "## Imports and Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AGhFQLfKRmyF",
    "outputId": "e71f8681-8c57-4c56-ac91-1c5ffaf9518e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from difflib import SequenceMatcher, HtmlDiff\n",
    "from IPython.display import HTML, display\n",
    "import nltk\n",
    "import hashlib\n",
    "\n",
    "nltk.download('punkt')\n",
    "\n",
    "BEGINNER_LEVEL_ONLY = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LkVuELopO3ME"
   },
   "outputs": [],
   "source": [
    "# Helper functions\n",
    "def getFormattedDiff(a, b):\n",
    "    \"\"\"\n",
    "        Given two strings (a, b), getFormattedDiff returns the HTML formatted strings (formatted_a, formatted_b)\n",
    "        Source: https://stackoverflow.com/questions/76527591/python-highlight-differences-between-two-strings-in-jupyter-notebook\n",
    "    \"\"\"\n",
    "    # highlight colors\n",
    "    color_delete = '#FBAE8A'  # Red highlight color for deletions\n",
    "    color_insert = '#9FFB8A'  # Green highlight color for insertions\n",
    "    color_replace = '#FBE78A' # Yellow highlight color for replacements\n",
    "\n",
    "    # the common format string used for highlighted segments\n",
    "    f_str = '<span style=\"background: {};\">{}</span>'\n",
    "\n",
    "    # initialize the sequence matcher\n",
    "    s = SequenceMatcher(None, a, b)\n",
    "\n",
    "    # stringbuilders for the formatted strings\n",
    "    formatted_a = []\n",
    "    formatted_b = []\n",
    "\n",
    "    # iterate through all char blocks\n",
    "    for tag, i1, i2, j1, j2 in s.get_opcodes():\n",
    "        if tag == 'equal':\n",
    "            # if the blovks are the same, append block to both strings without any formatting\n",
    "            formatted_a.append(a[i1:i2])\n",
    "            formatted_b.append(b[j1:j2])\n",
    "        elif tag == 'delete':\n",
    "            # if this is a deletion block, append block to the first string with the delete highlight\n",
    "            formatted_a.append(f_str.format(color_delete, a[i1:i2]))\n",
    "        elif tag == 'insert':\n",
    "            # if this is a insertion block, append block to the second string with the insert highlight\n",
    "            formatted_b.append(f_str.format(color_insert, b[j1:j2]))\n",
    "        elif tag == 'replace':\n",
    "            # if this is a replacement block, append block to both strings with the replace highlight\n",
    "            formatted_a.append(f_str.format(color_replace, a[i1:i2]))\n",
    "            formatted_b.append(f_str.format(color_replace, b[j1:j2]))\n",
    "\n",
    "    # return the formatted strings\n",
    "    return ''.join(formatted_a), ''.join(formatted_b)\n",
    "\n",
    "# Display diff of two columns in dataset\n",
    "def diff(df_to_diff, col1, col2, n=2, dropna=False, onlydiffs=True, colwidth=50):\n",
    "    data = df_to_diff[[col1, col2]]\n",
    "    if onlydiffs:\n",
    "        data = data[data[col1] != data[col2]]\n",
    "    if dropna:\n",
    "        data = data.dropna()\n",
    "    else:\n",
    "        data = data.fillna('')\n",
    "    samplesize = n if data.shape[0] >= n else data.shape[0]\n",
    "    data = data.sample(n = samplesize)\n",
    "    diff = HtmlDiff(wrapcolumn=colwidth).make_file(data[col1],data[col2], col1, col2)\n",
    "    display(HTML(diff))\n",
    "\n",
    "# dowload data\n",
    "def get_data(force_download=False):\n",
    "    lists = ['famous.F17.csv', 'famous.S17.csv', 'famous.S18.csv', 'famous.SU17.csv', 'famous.W18.csv',\n",
    "             'vacation.F17.csv', 'vacation.S17.csv', 'vacation.S18.csv', 'vacation.SU17.csv', 'vacation.W18.csv']\n",
    "    for file in lists:\n",
    "        if not os.path.isfile(file):\n",
    "            # File not there download it\n",
    "            os.system(f'wget https://raw.githubusercontent.com/ucdaviscl/cowsl2h/master/csv/{file}')\n",
    "        elif force_download:\n",
    "            # Remove existing file and re-download it.\n",
    "            os.remove(file)\n",
    "            os.system(f'wget https://raw.githubusercontent.com/ucdaviscl/cowsl2h/master/csv/{file}')\n",
    "\n",
    "    df = pd.concat([pd.read_csv(file) for file in lists], axis=0)\n",
    "    return df\n",
    "\n",
    "# use nltk for better spliting the sentence\n",
    "def split_paragraph_with_nltk(text):\n",
    "  if type(text) == str:\n",
    "    combine_sentence = []\n",
    "    for i in text.split('\\n\\n\\xa0\\n\\n'):\n",
    "      sentences = nltk.tokenize.sent_tokenize(i, language='spanish')\n",
    "      combine_sentence.append(sentences)\n",
    "    return [item for sublist in combine_sentence for item in sublist]\n",
    "  else:\n",
    "    return []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d9vrozIqlCC7",
    "tags": []
   },
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LciVv4rbR41h"
   },
   "outputs": [],
   "source": [
    "df = get_data(force_download=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Zo4wCSTBO3MF",
    "tags": []
   },
   "source": [
    "## Initial Sanity Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "SexDKbcCO3MF",
    "outputId": "33208b91-3389-4dae-fedb-2902f5ef56fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of essasys in original data: 1725\n",
      "\n",
      "Columns in the original data:\n",
      "Index(['Unnamed: 0', 'id', 'prompt', 'quarter', 'course', 'age', 'gender',\n",
      "       'l1 language', 'other l1 language(s)', 'language(s) used at home',\n",
      "       'language(s) studied', 'listening comprehension',\n",
      "       'reading comprehension', 'speaking ability', 'writing ability',\n",
      "       'study abroad', 'essay', 'a personal annotator1',\n",
      "       'a personal annotator2', 'gender-number annotator1',\n",
      "       'gender-number annotator2', 'corrected'],\n",
      "      dtype='object')\n",
      "\n",
      "Essays count at each course level:\n",
      "\n",
      "Sample Gender-Number annotations\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"\n",
       "          \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n",
       "\n",
       "<html>\n",
       "\n",
       "<head>\n",
       "    <meta http-equiv=\"Content-Type\"\n",
       "          content=\"text/html; charset=utf-8\" />\n",
       "    <title></title>\n",
       "    <style type=\"text/css\">\n",
       "        table.diff {font-family:Courier; border:medium;}\n",
       "        .diff_header {background-color:#e0e0e0}\n",
       "        td.diff_header {text-align:right}\n",
       "        .diff_next {background-color:#c0c0c0}\n",
       "        .diff_add {background-color:#aaffaa}\n",
       "        .diff_chg {background-color:#ffff77}\n",
       "        .diff_sub {background-color:#ffaaaa}\n",
       "    </style>\n",
       "</head>\n",
       "\n",
       "<body>\n",
       "    \n",
       "    <table class=\"diff\" id=\"difflib_chg_to0__top\"\n",
       "           cellspacing=\"0\" cellpadding=\"0\" rules=\"groups\" >\n",
       "        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n",
       "        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n",
       "        <thead><tr><th class=\"diff_next\"><br /></th><th colspan=\"2\" class=\"diff_header\">essay</th><th class=\"diff_next\"><br /></th><th colspan=\"2\" class=\"diff_header\">gender-number annotator2</th></tr></thead>\n",
       "        <tbody>\n",
       "            <tr><td class=\"diff_next\" id=\"difflib_chg_to0__0\"><a href=\"#difflib_chg_to0__top\">t</a></td><td class=\"diff_header\" id=\"from0_1\">1</td><td nowrap=\"nowrap\">Para&nbsp;mí,&nbsp;unas&nbsp;vacaciones&nbsp;perfectas&nbsp;es&nbsp;con&nbsp;todos&nbsp;los&nbsp;mie</td><td class=\"diff_next\"><a href=\"#difflib_chg_to0__top\">t</a></td><td class=\"diff_header\" id=\"to0_1\">1</td><td nowrap=\"nowrap\">Para&nbsp;mí,&nbsp;unas&nbsp;vacaciones&nbsp;perfectas&nbsp;es&nbsp;con&nbsp;todos&nbsp;los&nbsp;mie</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">mbros&nbsp;de&nbsp;mi&nbsp;familia&nbsp;y&nbsp;también&nbsp;mi&nbsp;mejor&nbsp;amiga.<span class=\"diff_chg\"> </span>Quiero&nbsp;vi</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">mbros&nbsp;de&nbsp;mi&nbsp;familia&nbsp;y&nbsp;también&nbsp;mi&nbsp;mejor&nbsp;amiga.<span class=\"diff_chg\">&nbsp;</span>Quiero&nbsp;vi</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ajar&nbsp;a&nbsp;un&nbsp;país&nbsp;tropical&nbsp;con&nbsp;todos&nbsp;ellos,&nbsp;o&nbsp;a&nbsp;la&nbsp;playa.<span class=\"diff_chg\"> </span></td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ajar&nbsp;a&nbsp;un&nbsp;país&nbsp;tropical&nbsp;con&nbsp;todos&nbsp;ellos,&nbsp;o&nbsp;a&nbsp;la&nbsp;playa.<span class=\"diff_chg\">&nbsp;</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\"></span>Por&nbsp;ejemplo,&nbsp;un&nbsp;viaje&nbsp;a&nbsp;Sri&nbsp;Lanka.<span class=\"diff_chg\"> </span>Sri&nbsp;Lanka&nbsp;tiene&nbsp;gran</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\"></span>Por&nbsp;ejemplo,&nbsp;un&nbsp;viaje&nbsp;a&nbsp;Sri&nbsp;Lanka.<span class=\"diff_chg\">&nbsp;</span>Sri&nbsp;Lanka&nbsp;tiene&nbsp;gran</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">des&nbsp;playas&nbsp;y&nbsp;bosques.<span class=\"diff_chg\"> </span>También&nbsp;me&nbsp;encanta&nbsp;la&nbsp;comida&nbsp;allí</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">des&nbsp;playas&nbsp;y&nbsp;bosques.<span class=\"diff_chg\">&nbsp;</span>También&nbsp;me&nbsp;encanta&nbsp;la&nbsp;comida&nbsp;allí</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">.<span class=\"diff_chg\"> </span>Cuando&nbsp;era&nbsp;joven,&nbsp;fui&nbsp;con&nbsp;mi&nbsp;familia.<span class=\"diff_chg\"> </span>\n",
       "\n",
       "Soy&nbsp;una&nbsp;<span class=\"diff_chg\">bucea</span></td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">.<span class=\"diff_chg\">&nbsp;</span>Cuando&nbsp;era&nbsp;joven,&nbsp;fui&nbsp;con&nbsp;mi&nbsp;familia.<span class=\"diff_chg\">&nbsp;</span>\n",
       "\n",
       "Soy&nbsp;una&nbsp;<span class=\"diff_chg\">[buce</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">dor&nbsp;scuba. </span>Por&nbsp;tanto&nbsp;amo&nbsp;estar&nbsp;cerca&nbsp;del&nbsp;océano.<span class=\"diff_chg\"> </span>Me&nbsp;enc</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">ador]{buceadora}&lt;ga:fm:adj:an&gt;&nbsp;scuba.&nbsp;</span>Por&nbsp;tanto&nbsp;amo&nbsp;est</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">antaba&nbsp;jugar&nbsp;en&nbsp;la&nbsp;playa&nbsp;con&nbsp;mi&nbsp;hermano,&nbsp;cuando&nbsp;era&nbsp;jov</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ar&nbsp;cerca&nbsp;del&nbsp;océano.<span class=\"diff_chg\">&nbsp;</span>Me&nbsp;encantaba&nbsp;jugar&nbsp;en&nbsp;la&nbsp;playa&nbsp;con</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">en.<span class=\"diff_chg\"> </span>Pero&nbsp;ahora,&nbsp;puedo&nbsp;disfrutar&nbsp;del&nbsp;sol&nbsp;y&nbsp;las&nbsp;bebidas&nbsp;c</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;mi&nbsp;hermano,&nbsp;cuando&nbsp;era&nbsp;joven.<span class=\"diff_chg\">&nbsp;</span>Pero&nbsp;ahora,&nbsp;puedo&nbsp;disfru</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">on&nbsp;mis&nbsp;amigos&nbsp;en&nbsp;la&nbsp;playa.<span class=\"diff_chg\"> </span>Mi&nbsp;mejor&nbsp;amiga&nbsp;es&nbsp;muy&nbsp;gracio</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">tar&nbsp;del&nbsp;sol&nbsp;y&nbsp;las&nbsp;bebidas&nbsp;con&nbsp;mis&nbsp;amigos&nbsp;en&nbsp;la&nbsp;playa.<span class=\"diff_chg\">&nbsp;</span>M</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">sa&nbsp;y&nbsp;a&nbsp;mi&nbsp;familia&nbsp;le&nbsp;gusta.<span class=\"diff_chg\"> </span>\n",
       "\n",
       "La&nbsp;comida&nbsp;picante&nbsp;y&nbsp;las&nbsp;f</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">i&nbsp;mejor&nbsp;amiga&nbsp;es&nbsp;muy&nbsp;graciosa&nbsp;y&nbsp;a&nbsp;mi&nbsp;familia&nbsp;le&nbsp;gusta.<span class=\"diff_chg\">&nbsp;</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">rutas&nbsp;tropicales&nbsp;de&nbsp;Sri&nbsp;Lanka&nbsp;son&nbsp;increíbles.&nbsp;Incluso&nbsp;l</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\"></span>\n",
       "\n",
       "La&nbsp;comida&nbsp;picante&nbsp;y&nbsp;las&nbsp;frutas&nbsp;tropicales&nbsp;de&nbsp;Sri&nbsp;Lank</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">os&nbsp;masajes&nbsp;son&nbsp;baratos&nbsp;y&nbsp;muy&nbsp;relajantes.<span class=\"diff_chg\"> </span>Los&nbsp;viajes&nbsp;con</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">a&nbsp;son&nbsp;increíbles.&nbsp;Incluso&nbsp;los&nbsp;masajes&nbsp;son&nbsp;baratos&nbsp;y&nbsp;muy</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;la&nbsp;familia&nbsp;son&nbsp;grandes&nbsp;porque&nbsp;tienen&nbsp;cuidado&nbsp;de&nbsp;mí.&nbsp;Ta</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;relajantes.<span class=\"diff_chg\">&nbsp;</span>Los&nbsp;viajes&nbsp;con&nbsp;la&nbsp;familia&nbsp;son&nbsp;grandes&nbsp;porq</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">mbién,&nbsp;Sri&nbsp;Lanka&nbsp;está&nbsp;muy&nbsp;cerca&nbsp;de&nbsp;India&nbsp;por&nbsp;tanto&nbsp;es&nbsp;f</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ue&nbsp;tienen&nbsp;cuidado&nbsp;de&nbsp;mí.&nbsp;También,&nbsp;Sri&nbsp;Lanka&nbsp;está&nbsp;muy&nbsp;ce</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ácil&nbsp;viajar&nbsp;allá.<span class=\"diff_sub\">\n",
       "\n",
       " </span></td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">rca&nbsp;de&nbsp;India&nbsp;por&nbsp;tanto&nbsp;es&nbsp;fácil&nbsp;viajar&nbsp;allá.</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\" id=\"from0_2\">2</td><td nowrap=\"nowrap\">El&nbsp;verano&nbsp;es&nbsp;la&nbsp;mejor&nbsp;temporada&nbsp;para&nbsp;viajar.&nbsp;Mis&nbsp;vacaci</td><td class=\"diff_next\"></td><td class=\"diff_header\" id=\"to0_2\">2</td><td nowrap=\"nowrap\">El&nbsp;verano&nbsp;es&nbsp;la&nbsp;mejor&nbsp;temporada&nbsp;para&nbsp;viajar.&nbsp;Mis&nbsp;vacaci</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ones&nbsp;perfectas&nbsp;sería&nbsp;un&nbsp;viaje&nbsp;a&nbsp;España.&nbsp;Hay&nbsp;muchas&nbsp;cosa</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ones&nbsp;perfectas&nbsp;sería&nbsp;un&nbsp;viaje&nbsp;a&nbsp;España.&nbsp;Hay&nbsp;muchas&nbsp;cosa</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">s&nbsp;que&nbsp;hacer&nbsp;en&nbsp;España.&nbsp;Usted&nbsp;puede&nbsp;disfrutar&nbsp;de&nbsp;la&nbsp;comi</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">s&nbsp;que&nbsp;hacer&nbsp;en&nbsp;España.&nbsp;Usted&nbsp;puede&nbsp;disfrutar&nbsp;de&nbsp;la&nbsp;comi</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">da&nbsp;española,&nbsp;música&nbsp;y&nbsp;festivals.&nbsp;España&nbsp;es&nbsp;un&nbsp;país&nbsp;herm</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">da&nbsp;española,&nbsp;música&nbsp;y&nbsp;festivals.&nbsp;España&nbsp;es&nbsp;un&nbsp;país&nbsp;herm</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">oso&nbsp;con&nbsp;playas&nbsp;increíbles&nbsp;también.&nbsp;España&nbsp;está&nbsp;situada&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">oso&nbsp;con&nbsp;playas&nbsp;increíbles&nbsp;también.&nbsp;España&nbsp;está&nbsp;situada&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">en&nbsp;el&nbsp;mar&nbsp;Mediterráneo.&nbsp;Por&nbsp;la&nbsp;mañana&nbsp;yo&nbsp;disfrutaría&nbsp;de</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">en&nbsp;el&nbsp;mar&nbsp;Mediterráneo.&nbsp;Por&nbsp;la&nbsp;mañana&nbsp;yo&nbsp;disfrutaría&nbsp;de</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;una&nbsp;taza&nbsp;de&nbsp;café&nbsp;y&nbsp;leería&nbsp;el&nbsp;periódico.&nbsp;Esto&nbsp;es&nbsp;mi&nbsp;mañ</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;una&nbsp;taza&nbsp;de&nbsp;café&nbsp;y&nbsp;leería&nbsp;el&nbsp;periódico.&nbsp;Esto&nbsp;es&nbsp;mi&nbsp;mañ</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ana&nbsp;ideal.&nbsp;Por&nbsp;la&nbsp;tarde,&nbsp;yo&nbsp;iría&nbsp;a&nbsp;la&nbsp;playa&nbsp;y&nbsp;tomar&nbsp;el&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ana&nbsp;ideal.&nbsp;Por&nbsp;la&nbsp;tarde,&nbsp;yo&nbsp;iría&nbsp;a&nbsp;la&nbsp;playa&nbsp;y&nbsp;tomar&nbsp;el&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">sol.&nbsp;La&nbsp;arena&nbsp;está&nbsp;siempre&nbsp;caliente&nbsp;y&nbsp;el&nbsp;agua&nbsp;está&nbsp;siem</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">sol.&nbsp;La&nbsp;arena&nbsp;está&nbsp;siempre&nbsp;caliente&nbsp;y&nbsp;el&nbsp;agua&nbsp;está&nbsp;siem</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">pre&nbsp;frío<span class=\"diff_chg\">.</span>&nbsp;Me&nbsp;gustaría&nbsp;abrir&nbsp;una&nbsp;cerveza&nbsp;fría&nbsp;y&nbsp;disfruta</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">pre&nbsp;<span class=\"diff_add\">[</span>frío<span class=\"diff_chg\">]{fría}&lt;ga:fm:adj:inan&gt;.</span>&nbsp;Me&nbsp;gustaría&nbsp;abrir&nbsp;una</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">r&nbsp;de&nbsp;la&nbsp;tarde.&nbsp;Por&nbsp;la&nbsp;noche&nbsp;iría&nbsp;a&nbsp;un&nbsp;partido&nbsp;de&nbsp;la&nbsp;cal</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;cerveza&nbsp;fría&nbsp;y&nbsp;disfrutar&nbsp;de&nbsp;la&nbsp;tarde.&nbsp;Por&nbsp;la&nbsp;noche&nbsp;irí</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">le&nbsp;o&nbsp;visitaría&nbsp;las&nbsp;barras&nbsp;locales.&nbsp;Los&nbsp;españoles&nbsp;les&nbsp;en</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">a&nbsp;a&nbsp;un&nbsp;partido&nbsp;de&nbsp;la&nbsp;calle&nbsp;o&nbsp;visitaría&nbsp;las&nbsp;barras&nbsp;local</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">canta&nbsp;celebrar&nbsp;ocasiones&nbsp;especiales&nbsp;con&nbsp;fiestas&nbsp;y&nbsp;alime</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">es.&nbsp;Los&nbsp;españoles&nbsp;les&nbsp;encanta&nbsp;celebrar&nbsp;ocasiones&nbsp;especi</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ntos.\n",
       "\n",
       "Si&nbsp;no&nbsp;te&nbsp;gusta&nbsp;la&nbsp;playa,&nbsp;te&nbsp;puedes&nbsp;viajar&nbsp;a&nbsp;la&nbsp;c</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ales&nbsp;con&nbsp;fiestas&nbsp;y&nbsp;alimentos.\n",
       "\n",
       "Si&nbsp;no&nbsp;te&nbsp;gusta&nbsp;la&nbsp;playa,</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">iudad.&nbsp;Barcelona&nbsp;es&nbsp;una&nbsp;de&nbsp;mis&nbsp;ciudades&nbsp;favoritas.&nbsp;Pued</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;te&nbsp;puedes&nbsp;viajar&nbsp;a&nbsp;la&nbsp;ciudad.&nbsp;Barcelona&nbsp;es&nbsp;una&nbsp;de&nbsp;mis&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">es&nbsp;ver&nbsp;la&nbsp;famosa&nbsp;basílica,&nbsp;La&nbsp;Sagrada&nbsp;Familia.&nbsp;Aún&nbsp;no&nbsp;e</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ciudades&nbsp;favoritas.&nbsp;Puedes&nbsp;ver&nbsp;la&nbsp;famosa&nbsp;basílica,&nbsp;La&nbsp;S</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">stá&nbsp;<span class=\"diff_chg\">completado</span>,&nbsp;pero&nbsp;sigue&nbsp;siendo&nbsp;impresionante.&nbsp;Barcel</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">agrada&nbsp;Familia.&nbsp;Aún&nbsp;no&nbsp;está&nbsp;<span class=\"diff_chg\">[completado]{completada}&lt;ga</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ona&nbsp;tiene&nbsp;un&nbsp;montón&nbsp;de&nbsp;arquitectura&nbsp;para&nbsp;turistas&nbsp;a&nbsp;mir</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">:fm:adj:inan&gt;</span>,&nbsp;pero&nbsp;sigue&nbsp;siendo&nbsp;impresionante.&nbsp;Barcelo</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ar.&nbsp;<span class=\"diff_chg\"> </span>La&nbsp;ciudad&nbsp;es&nbsp;famosa&nbsp;por&nbsp;su&nbsp;arquitectura.&nbsp;Por&nbsp;eso&nbsp;e</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">na&nbsp;tiene&nbsp;un&nbsp;montón&nbsp;de&nbsp;arquitectura&nbsp;para&nbsp;turistas&nbsp;a&nbsp;mira</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">s&nbsp;uno&nbsp;de&nbsp;mis&nbsp;lugares&nbsp;favoritos&nbsp;para&nbsp;visitar.\n",
       "\n",
       "Mis&nbsp;abuel</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">r.&nbsp;<span class=\"diff_chg\">&nbsp;</span>La&nbsp;ciudad&nbsp;es&nbsp;famosa&nbsp;por&nbsp;su&nbsp;arquitectura.&nbsp;Por&nbsp;eso&nbsp;es</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">os&nbsp;viven&nbsp;en&nbsp;España.&nbsp;Ellos&nbsp;viven&nbsp;en&nbsp;<span class=\"diff_chg\">un&nbsp;</span>pequeño<span class=\"diff_chg\">&nbsp;ciudad&nbsp;ll</span></td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;uno&nbsp;de&nbsp;mis&nbsp;lugares&nbsp;favoritos&nbsp;para&nbsp;visitar.\n",
       "\n",
       "Mis&nbsp;abuelo</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">amado</span>&nbsp;*CITY*.&nbsp;La&nbsp;casa&nbsp;de&nbsp;mis&nbsp;abuelos&nbsp;está&nbsp;muy&nbsp;cerca&nbsp;del</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">s&nbsp;viven&nbsp;en&nbsp;España.&nbsp;Ellos&nbsp;viven&nbsp;en&nbsp;<span class=\"diff_chg\">[un]{una}&lt;ga:fm:det:i</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;océano.&nbsp;Mis&nbsp;abuelos&nbsp;están&nbsp;envejeciendo,&nbsp;sería&nbsp;agradabl</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">nan&gt;&nbsp;[</span>pequeño<span class=\"diff_chg\">]{pequeña}&lt;ga:fm:adj:inan&gt;&nbsp;ciudad&lt;gat:noun</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">e&nbsp;verlos&nbsp;otra&nbsp;vez.&nbsp;Ellos&nbsp;son&nbsp;la&nbsp;razón&nbsp;principal&nbsp;por&nbsp;qué</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">:inan&gt;&nbsp;[llamado]{llamada}&lt;ga:fm:adj:inan&gt;</span>&nbsp;*CITY*.&nbsp;La&nbsp;ca</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;yo&nbsp;iría&nbsp;a&nbsp;España.&nbsp;Mi&nbsp;familia&nbsp;es&nbsp;muy&nbsp;importante&nbsp;para&nbsp;mí</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">sa&nbsp;de&nbsp;mis&nbsp;abuelos&nbsp;está&nbsp;muy&nbsp;cerca&nbsp;del&nbsp;océano.&nbsp;Mis&nbsp;abuelo</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">,&nbsp;y&nbsp;<span class=\"diff_chg\">esto</span>&nbsp;sería&nbsp;las&nbsp;vacaciones&nbsp;perfectas.&nbsp;Ahora&nbsp;necesito</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">s&nbsp;están&nbsp;envejeciendo,&nbsp;sería&nbsp;agradable&nbsp;verlos&nbsp;otra&nbsp;vez.&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;dinero&nbsp;para&nbsp;viajar&nbsp;a&nbsp;España.</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">Ellos&nbsp;son&nbsp;la&nbsp;razón&nbsp;principal&nbsp;por&nbsp;qué&nbsp;yo&nbsp;iría&nbsp;a&nbsp;España.&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\"></td><td nowrap=\"nowrap\">&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">Mi&nbsp;familia&nbsp;es&nbsp;muy&nbsp;importante&nbsp;para&nbsp;mí,&nbsp;y&nbsp;<span class=\"diff_chg\">[esto]{estas}&lt;g</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\"></td><td nowrap=\"nowrap\">&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">a:na:fm:ps:pron:inan&gt;</span>&nbsp;sería&nbsp;las&nbsp;vacaciones&nbsp;perfectas.&nbsp;A</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\"></td><td nowrap=\"nowrap\">&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">hora&nbsp;necesito&nbsp;dinero&nbsp;para&nbsp;viajar&nbsp;a&nbsp;España.</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\" id=\"from0_3\">3</td><td nowrap=\"nowrap\">Si&nbsp;pudiera&nbsp;planear&nbsp;unas&nbsp;vacaciones&nbsp;para&nbsp;ir&nbsp;a&nbsp;cualquier&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\" id=\"to0_3\">3</td><td nowrap=\"nowrap\">Si&nbsp;pudiera&nbsp;planear&nbsp;unas&nbsp;vacaciones&nbsp;para&nbsp;ir&nbsp;a&nbsp;cualquier&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">parte&nbsp;del&nbsp;mundo,&nbsp;planearía&nbsp;un&nbsp;viaje&nbsp;a&nbsp;Cancún,&nbsp;México.&nbsp;A</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">parte&nbsp;del&nbsp;mundo,&nbsp;planearía&nbsp;un&nbsp;viaje&nbsp;a&nbsp;Cancún,&nbsp;México.&nbsp;A</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">unque&nbsp;he&nbsp;vivido&nbsp;en&nbsp;*STATE*&nbsp;toda&nbsp;mi&nbsp;vida,&nbsp;nunca&nbsp;he&nbsp;visit</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">unque&nbsp;he&nbsp;vivido&nbsp;en&nbsp;*STATE*&nbsp;toda&nbsp;mi&nbsp;vida,&nbsp;nunca&nbsp;he&nbsp;visit</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ado&nbsp;México.&nbsp;Este&nbsp;es&nbsp;un&nbsp;viaje&nbsp;que&nbsp;espero&nbsp;tener&nbsp;en&nbsp;un&nbsp;fut</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ado&nbsp;México.&nbsp;Este&nbsp;es&nbsp;un&nbsp;viaje&nbsp;que&nbsp;espero&nbsp;tener&nbsp;en&nbsp;un&nbsp;fut</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">uro&nbsp;próximo.&nbsp;Para&nbsp;este&nbsp;viaje,&nbsp;primero&nbsp;conduciría&nbsp;al&nbsp;sur</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">uro&nbsp;próximo.&nbsp;Para&nbsp;este&nbsp;viaje,&nbsp;primero&nbsp;conduciría&nbsp;al&nbsp;sur</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;de&nbsp;California.&nbsp;Estoy&nbsp;planeando&nbsp;ir&nbsp;en&nbsp;este&nbsp;viaje&nbsp;con&nbsp;mi</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;de&nbsp;California.&nbsp;Estoy&nbsp;planeando&nbsp;ir&nbsp;en&nbsp;este&nbsp;viaje&nbsp;con&nbsp;mi</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;mejor&nbsp;amigo&nbsp;que&nbsp;vive&nbsp;en&nbsp;el&nbsp;sur&nbsp;de&nbsp;California,&nbsp;así&nbsp;que&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;mejor&nbsp;amigo&nbsp;que&nbsp;vive&nbsp;en&nbsp;el&nbsp;sur&nbsp;de&nbsp;California,&nbsp;así&nbsp;que&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">primero&nbsp;reunirse&nbsp;con&nbsp;<span class=\"diff_chg\">ella&nbsp;allí,&nbsp;así</span>&nbsp;que&nbsp;podemos&nbsp;ir&nbsp;junt</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">primero&nbsp;reunirse&nbsp;con&nbsp;<span class=\"diff_chg\">[ella]{él}&lt;ga:mf:pron:an&gt;&nbsp;allí,&nbsp;as</span></td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">os.&nbsp;Me&nbsp;gustaría&nbsp;pasar&nbsp;un&nbsp;par&nbsp;de&nbsp;días&nbsp;allí&nbsp;explorando&nbsp;lo</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\"><span class=\"diff_chg\">í</span>&nbsp;que&nbsp;podemos&nbsp;ir&nbsp;juntos.&nbsp;Me&nbsp;gustaría&nbsp;pasar&nbsp;un&nbsp;par&nbsp;de&nbsp;dí</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">s&nbsp;Ángeles,&nbsp;ir&nbsp;a&nbsp;las&nbsp;playas,&nbsp;y&nbsp;comer&nbsp;la&nbsp;comida&nbsp;increíble</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">as&nbsp;allí&nbsp;explorando&nbsp;los&nbsp;Ángeles,&nbsp;ir&nbsp;a&nbsp;las&nbsp;playas,&nbsp;y&nbsp;come</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;que&nbsp;tienen&nbsp;allí.&nbsp;Después,&nbsp;volamos&nbsp;a&nbsp;Cancún.\n",
       "\n",
       "En&nbsp;Cancún</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">r&nbsp;la&nbsp;comida&nbsp;increíble&nbsp;que&nbsp;tienen&nbsp;allí.&nbsp;Después,&nbsp;volamos</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">,&nbsp;primero&nbsp;iba&nbsp;a&nbsp;ver&nbsp;el&nbsp;Chichén&nbsp;Itzá,&nbsp;que&nbsp;es&nbsp;una&nbsp;de&nbsp;las&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;a&nbsp;Cancún.\n",
       "\n",
       "En&nbsp;Cancún,&nbsp;primero&nbsp;iba&nbsp;a&nbsp;ver&nbsp;el&nbsp;Chichén&nbsp;Itz</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ruinas&nbsp;mayas&nbsp;antiguas&nbsp;realmente&nbsp;grandes.&nbsp;Luego,&nbsp;me&nbsp;gust</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">á,&nbsp;que&nbsp;es&nbsp;una&nbsp;de&nbsp;las&nbsp;ruinas&nbsp;mayas&nbsp;antiguas&nbsp;realmente&nbsp;gr</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">aría&nbsp;ir&nbsp;a&nbsp;una&nbsp;de&nbsp;sus&nbsp;muchas&nbsp;playas&nbsp;e&nbsp;ir&nbsp;a&nbsp;bucear&nbsp;y&nbsp;buce</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">andes.&nbsp;Luego,&nbsp;me&nbsp;gustaría&nbsp;ir&nbsp;a&nbsp;una&nbsp;de&nbsp;sus&nbsp;muchas&nbsp;playas</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ar&nbsp;con&nbsp;los&nbsp;delfines.&nbsp;Cancún&nbsp;también&nbsp;es&nbsp;conocido&nbsp;por&nbsp;su&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;e&nbsp;ir&nbsp;a&nbsp;bucear&nbsp;y&nbsp;bucear&nbsp;con&nbsp;los&nbsp;delfines.&nbsp;Cancún&nbsp;tambié</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">gran&nbsp;vida&nbsp;nocturna.&nbsp;Si&nbsp;yo&nbsp;fuera&nbsp;a&nbsp;ir&nbsp;allí,&nbsp;me&nbsp;volvería&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">n&nbsp;es&nbsp;conocido&nbsp;por&nbsp;su&nbsp;gran&nbsp;vida&nbsp;nocturna.&nbsp;Si&nbsp;yo&nbsp;fuera&nbsp;a&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">a&nbsp;la&nbsp;Coco&nbsp;Bongo.&nbsp;Es&nbsp;un&nbsp;gran&nbsp;club&nbsp;nocturno&nbsp;con&nbsp;actuacion</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ir&nbsp;allí,&nbsp;me&nbsp;volvería&nbsp;a&nbsp;la&nbsp;Coco&nbsp;Bongo.&nbsp;Es&nbsp;un&nbsp;gran&nbsp;club&nbsp;n</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">es&nbsp;de&nbsp;acróbatas,&nbsp;líneas&nbsp;Conga,&nbsp;bandas&nbsp;en&nbsp;vivo&nbsp;y&nbsp;imitado</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">octurno&nbsp;con&nbsp;actuaciones&nbsp;de&nbsp;acróbatas,&nbsp;líneas&nbsp;Conga,&nbsp;ban</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">res&nbsp;de&nbsp;músicos.&nbsp;Una&nbsp;noche&nbsp;típica&nbsp;en&nbsp;Coco&nbsp;Bongo&nbsp;incluye&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">das&nbsp;en&nbsp;vivo&nbsp;y&nbsp;imitadores&nbsp;de&nbsp;músicos.&nbsp;Una&nbsp;noche&nbsp;típica&nbsp;e</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">videos&nbsp;proyectados,&nbsp;globos,&nbsp;burbujas&nbsp;de&nbsp;jabón&nbsp;y&nbsp;confeti</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">n&nbsp;Coco&nbsp;Bongo&nbsp;incluye&nbsp;videos&nbsp;proyectados,&nbsp;globos,&nbsp;burbuj</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">.&nbsp;Por&nbsp;último,&nbsp;me&nbsp;gustaría&nbsp;visitar&nbsp;isla&nbsp;mujeres&nbsp;que&nbsp;es&nbsp;u</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">as&nbsp;de&nbsp;jabón&nbsp;y&nbsp;confeti.&nbsp;Por&nbsp;último,&nbsp;me&nbsp;gustaría&nbsp;visitar&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">na&nbsp;de&nbsp;las&nbsp;playas&nbsp;más&nbsp;famosas&nbsp;de&nbsp;Cancún.&nbsp;Isla&nbsp;mujeres&nbsp;es</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">isla&nbsp;mujeres&nbsp;que&nbsp;es&nbsp;una&nbsp;de&nbsp;las&nbsp;playas&nbsp;más&nbsp;famosas&nbsp;de&nbsp;Ca</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;un&nbsp;lugar&nbsp;muy&nbsp;relajante&nbsp;que&nbsp;se&nbsp;puede&nbsp;llegar&nbsp;a&nbsp;través&nbsp;de</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">ncún.&nbsp;Isla&nbsp;mujeres&nbsp;es&nbsp;un&nbsp;lugar&nbsp;muy&nbsp;relajante&nbsp;que&nbsp;se&nbsp;pue</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;un&nbsp;corto&nbsp;paseo&nbsp;en&nbsp;barco.&nbsp;Allí&nbsp;me&nbsp;gustaría&nbsp;disfrutar&nbsp;de</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">de&nbsp;llegar&nbsp;a&nbsp;través&nbsp;de&nbsp;un&nbsp;corto&nbsp;paseo&nbsp;en&nbsp;barco.&nbsp;Allí&nbsp;me&nbsp;</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;los&nbsp;espectáculos&nbsp;de&nbsp;mar,&nbsp;Zip&nbsp;line&nbsp;a&nbsp;través&nbsp;de&nbsp;la&nbsp;isla&nbsp;</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">gustaría&nbsp;disfrutar&nbsp;de&nbsp;los&nbsp;espectáculos&nbsp;de&nbsp;mar,&nbsp;Zip&nbsp;line</td></tr>\n",
       "            <tr><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">y&nbsp;bucear&nbsp;en&nbsp;el&nbsp;tiburón.</td><td class=\"diff_next\"></td><td class=\"diff_header\">></td><td nowrap=\"nowrap\">&nbsp;a&nbsp;través&nbsp;de&nbsp;la&nbsp;isla&nbsp;y&nbsp;bucear&nbsp;en&nbsp;el&nbsp;tiburón.</td></tr>\n",
       "        </tbody>\n",
       "    </table>\n",
       "    <table class=\"diff\" summary=\"Legends\">\n",
       "        <tr> <th colspan=\"2\"> Legends </th> </tr>\n",
       "        <tr> <td> <table border=\"\" summary=\"Colors\">\n",
       "                      <tr><th> Colors </th> </tr>\n",
       "                      <tr><td class=\"diff_add\">&nbsp;Added&nbsp;</td></tr>\n",
       "                      <tr><td class=\"diff_chg\">Changed</td> </tr>\n",
       "                      <tr><td class=\"diff_sub\">Deleted</td> </tr>\n",
       "                  </table></td>\n",
       "             <td> <table border=\"\" summary=\"Links\">\n",
       "                      <tr><th colspan=\"2\"> Links </th> </tr>\n",
       "                      <tr><td>(f)irst change</td> </tr>\n",
       "                      <tr><td>(n)ext change</td> </tr>\n",
       "                      <tr><td>(t)op</td> </tr>\n",
       "                  </table></td> </tr>\n",
       "    </table>\n",
       "</body>\n",
       "\n",
       "</html>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(f'Number of essasys in original data: {df.shape[0]}')\n",
    "print()\n",
    "print(f'Columns in the original data:')\n",
    "print(df.columns)\n",
    "print()\n",
    "print(f'Essays count at each course level:')\n",
    "df['course'].value_counts()\n",
    "print()\n",
    "print(\"Sample Gender-Number annotations\")\n",
    "diff(df, 'essay', 'gender-number annotator2', n=3, dropna=False, colwidth=55, onlydiffs=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8VmFJRcpO3MF",
    "tags": []
   },
   "source": [
    "## Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4szLr1zxO3MG"
   },
   "source": [
    "### Projection and filtering of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GyN5OyCPO3MG",
    "outputId": "278cf6a7-d1ad-4c90-c167-9713fed5150e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-4-42b69671fcc0>:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['paragraph_id'] = range(1,len(df)+1)\n"
     ]
    }
   ],
   "source": [
    "# Get rid of unneeded columns and rows\n",
    "df = df[['id', 'prompt', 'quarter', 'course', 'essay', 'a personal annotator1',\n",
    "   'a personal annotator2', 'gender-number annotator1',\n",
    "   'gender-number annotator2']]\n",
    "\n",
    "# add paragraph id\n",
    "df['paragraph_id'] = range(1,len(df)+1)\n",
    "\n",
    "if BEGINNER_LEVEL_ONLY:\n",
    "    # Keep only beginner essays\n",
    "    df = df[(df['course'] == 'SPA 1') | (df['course'] == 'SPA 2') | (df['course'] == 'SPA 3')]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6iqtbCnBO3MG"
   },
   "source": [
    "### Clean up messy format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6uyJ0tscO3MG"
   },
   "outputs": [],
   "source": [
    "def cleanup(data):\n",
    "    patterns = { '\\n\\xa0\\n\\n':'\\n\\n\\xa0\\n\\n',\n",
    "                '\\n\\n\\xa0':'\\n\\n\\xa0\\n\\n',\n",
    "                '\\n\\n\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0':'\\n\\n\\xa0\\n\\n',\n",
    "                '\\n\\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0': '\\n\\n\\xa0\\n\\n'\n",
    "    }\n",
    "    df_clean = data\n",
    "    df_clean['essay'].replace(patterns, regex= True, inplace = True)\n",
    "    df_clean['a personal annotator1'].replace(patterns, regex= True, inplace = True)\n",
    "    df_clean['a personal annotator2'].replace(patterns, regex= True, inplace = True)\n",
    "    df_clean['gender-number annotator1'].replace(patterns, regex= True, inplace = True)\n",
    "    df_clean['gender-number annotator2'].replace(patterns, regex= True, inplace = True)\n",
    "    return df_clean\n",
    "\n",
    "df = cleanup(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "A4OSDfhTO3MG"
   },
   "source": [
    "### Drop duplicated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 133
    },
    "id": "AyzIIIm3O3MG",
    "outputId": "32164445-f805-4279-b5bf-220e003a594f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-07796825-2469-47e0-8d2a-0073301394ad\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prompt</th>\n",
       "      <th>quarter</th>\n",
       "      <th>course</th>\n",
       "      <th>essay</th>\n",
       "      <th>a personal annotator1</th>\n",
       "      <th>a personal annotator2</th>\n",
       "      <th>gender-number annotator1</th>\n",
       "      <th>gender-number annotator2</th>\n",
       "      <th>paragraph_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>150930</td>\n",
       "      <td>famous</td>\n",
       "      <td>S17</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Jennifer Lopez es una cantante famosa. Su nomb...</td>\n",
       "      <td>Jennifer Lopez es una cantante famosa. Su nomb...</td>\n",
       "      <td>Jennifer Lopez es una cantante famosa. Su nomb...</td>\n",
       "      <td>Jennifer Lopez es una cantante famosa. Su nomb...</td>\n",
       "      <td>Jennifer Lopez es una cantante famosa. Su nomb...</td>\n",
       "      <td>217</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-07796825-2469-47e0-8d2a-0073301394ad')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-07796825-2469-47e0-8d2a-0073301394ad button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-07796825-2469-47e0-8d2a-0073301394ad');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "        id  prompt quarter course  \\\n",
       "41  150930  famous     S17  SPA 2   \n",
       "\n",
       "                                                essay  \\\n",
       "41  Jennifer Lopez es una cantante famosa. Su nomb...   \n",
       "\n",
       "                                a personal annotator1  \\\n",
       "41  Jennifer Lopez es una cantante famosa. Su nomb...   \n",
       "\n",
       "                                a personal annotator2  \\\n",
       "41  Jennifer Lopez es una cantante famosa. Su nomb...   \n",
       "\n",
       "                             gender-number annotator1  \\\n",
       "41  Jennifer Lopez es una cantante famosa. Su nomb...   \n",
       "\n",
       "                             gender-number annotator2  paragraph_id  \n",
       "41  Jennifer Lopez es una cantante famosa. Su nomb...           217  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['essay'].duplicated()==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5ZaMCOURO3MH"
   },
   "outputs": [],
   "source": [
    "# drop duplicated data\n",
    "df = df[(~df['essay'].duplicated())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WNdQqjfCO3MH"
   },
   "source": [
    "## Add computed cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "83WQnGRkO3MH"
   },
   "outputs": [],
   "source": [
    "# Add error counts\n",
    "def add_error_counts(data):\n",
    "    pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "\n",
    "    data['p_anno1_error_cnt'] = data['a personal annotator1'].str.count(pattern)\n",
    "    data['p_anno2_error_cnt'] = data['a personal annotator2'].str.count(pattern)\n",
    "    data['g_anno1_error_cnt'] = data['gender-number annotator1'].str.count(pattern)\n",
    "    data['g_anno2_error_cnt'] = data['gender-number annotator2'].str.count(pattern)\n",
    "\n",
    "    data['p_anno1_error_cnt'] = data['p_anno1_error_cnt'].fillna(0)\n",
    "    data['p_anno2_error_cnt'] = data['p_anno2_error_cnt'].fillna(0)\n",
    "    data['g_anno1_error_cnt'] = data['g_anno1_error_cnt'].fillna(0)\n",
    "    data['g_anno2_error_cnt'] = data['g_anno2_error_cnt'].fillna(0)\n",
    "    data = data.astype({'p_anno1_error_cnt': int, 'p_anno2_error_cnt': int, 'g_anno1_error_cnt': int, 'g_anno2_error_cnt': int})\n",
    "    return data\n",
    "\n",
    "df = add_error_counts(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jpiU4Q8nO3MH"
   },
   "outputs": [],
   "source": [
    "# Pick annotation with most error between the two annotated columns for both personal and gender\n",
    "df['p_annotator_sel']= df.apply(lambda row: row['a personal annotator1'] if row['p_anno1_error_cnt'] >= row['p_anno2_error_cnt'] else row['a personal annotator2'], axis = 1)\n",
    "df['g_annotator_sel']= df.apply(lambda row: row['gender-number annotator1'] if row['g_anno1_error_cnt'] >= row['g_anno2_error_cnt'] else row['gender-number annotator2'], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "weyK3GNqO3MH"
   },
   "source": [
    "## Split the paragraph into sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RjyrUywKO3MH",
    "outputId": "f31a245d-f032-420f-888d-70bd4a427335"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(35, 22)\n",
      "(43, 22)\n"
     ]
    }
   ],
   "source": [
    "#split into list\n",
    "df['essay_new'] = df['essay'].apply(split_paragraph_with_nltk)\n",
    "df['p_annotator'] = df['p_annotator_sel'].apply(split_paragraph_with_nltk)\n",
    "df['g_annotator'] = df['g_annotator_sel'].apply(split_paragraph_with_nltk)\n",
    "\n",
    "#check if the paragraph sentence count are the same\n",
    "df['p_annotator_cnt'] =  df['p_annotator'].apply(lambda x: len(x))\n",
    "df['g_annotator_cnt'] =  df['g_annotator'].apply(lambda x: len(x))\n",
    "df['essay_new_cnt'] =  df['essay_new'].apply(lambda x: len(x))\n",
    "\n",
    "print(df[df['essay_new_cnt'] != df['p_annotator_cnt']].shape)\n",
    "print(df[df['essay_new_cnt'] != df['g_annotator_cnt']].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5_fgZLnYo290"
   },
   "source": [
    "### Summary of cleaned data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nXQkbmH0r68a",
    "outputId": "b7bac850-fd59-4091-e714-3770075eb4c2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 1048 beginner essays on \"famous\" and \"vacation\" prompts in the dataset.\n"
     ]
    }
   ],
   "source": [
    "print(f'We have {df.shape[0]} beginner essays on \"famous\" and \"vacation\" prompts in the dataset.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xUmgBI40ydDu",
    "outputId": "9db6064f-f5de-4547-cc86-a4685b558c40"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-null values in columns\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "id                          1048\n",
       "prompt                      1048\n",
       "quarter                     1048\n",
       "course                      1048\n",
       "essay                       1048\n",
       "a personal annotator1       1047\n",
       "a personal annotator2        677\n",
       "gender-number annotator1    1043\n",
       "gender-number annotator2    1043\n",
       "paragraph_id                1048\n",
       "p_anno1_error_cnt           1048\n",
       "p_anno2_error_cnt           1048\n",
       "g_anno1_error_cnt           1048\n",
       "g_anno2_error_cnt           1048\n",
       "p_annotator_sel             1047\n",
       "g_annotator_sel             1043\n",
       "essay_new                   1048\n",
       "p_annotator                 1048\n",
       "g_annotator                 1048\n",
       "p_annotator_cnt             1048\n",
       "g_annotator_cnt             1048\n",
       "essay_new_cnt               1048\n",
       "dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Non-null values in columns')\n",
    "df[df['essay'].notnull()].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XUJcCJ75O3MI",
    "outputId": "463840e2-3cea-4a93-d6e6-660a9802a35d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values in each column\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "prompt                      2\n",
       "course                      3\n",
       "essay                    1048\n",
       "a personal annotator1    1047\n",
       "a personal annotator2     677\n",
       "p_annotator_sel          1047\n",
       "p_annotator_sel          1047\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Unique values in each column')\n",
    "df[['prompt', 'course', 'essay', 'a personal annotator1', 'a personal annotator2', 'p_annotator_sel', 'p_annotator_sel']].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 620
    },
    "id": "axJ3X1EERIPW",
    "outputId": "dad09e13-8516-44ee-83eb-70f455ee9a19"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-5db33e5e-e170-4794-b580-797ff7be9d28\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prompt</th>\n",
       "      <th>quarter</th>\n",
       "      <th>course</th>\n",
       "      <th>essay</th>\n",
       "      <th>a personal annotator1</th>\n",
       "      <th>a personal annotator2</th>\n",
       "      <th>gender-number annotator1</th>\n",
       "      <th>gender-number annotator2</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>...</th>\n",
       "      <th>g_anno1_error_cnt</th>\n",
       "      <th>g_anno2_error_cnt</th>\n",
       "      <th>p_annotator_sel</th>\n",
       "      <th>g_annotator_sel</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>p_annotator</th>\n",
       "      <th>g_annotator</th>\n",
       "      <th>p_annotator_cnt</th>\n",
       "      <th>g_annotator_cnt</th>\n",
       "      <th>essay_new_cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>157609</td>\n",
       "      <td>famous</td>\n",
       "      <td>S17</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>\\nMi atleta favorita es Rob Gronkowski, un jug...</td>\n",
       "      <td>Mi atleta favorita es Rob Gronkowski, un jugad...</td>\n",
       "      <td>Mi atleta favorita es Rob Gronkowski, un jugad...</td>\n",
       "      <td>Mi atleta [favorita]{favorito}&lt;ga:mf:adj:aty:a...</td>\n",
       "      <td>Mi atleta [favorita]{favorito}&lt;ga:mf:adj:aty:a...</td>\n",
       "      <td>339</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>Mi atleta favorita es Rob Gronkowski, un jugad...</td>\n",
       "      <td>Mi atleta [favorita]{favorito}&lt;ga:mf:adj:aty:a...</td>\n",
       "      <td>[\\nMi atleta favorita es Rob Gronkowski, un ju...</td>\n",
       "      <td>[Mi atleta favorita es Rob Gronkowski, un juga...</td>\n",
       "      <td>[Mi atleta [favorita]{favorito}&lt;ga:mf:adj:aty:...</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>125485</td>\n",
       "      <td>famous</td>\n",
       "      <td>S17</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>----------------------------------------------...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>381</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[---------------------------------------------...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>111462</td>\n",
       "      <td>vacation</td>\n",
       "      <td>S17</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Mi vacación perfecta consiste en muchas cosas ...</td>\n",
       "      <td>Mi vacación perfecta consiste en muchas cosas ...</td>\n",
       "      <td>Mi vacación perfecta consiste en muchas cosas ...</td>\n",
       "      <td>Mi [vacación]{vacaciones}&lt;in:s:noun:inan&gt; perf...</td>\n",
       "      <td>Mi [vacación]{vacaciones}&lt;in:s:noun:inan&gt; perf...</td>\n",
       "      <td>1119</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Mi vacación perfecta consiste en muchas cosas ...</td>\n",
       "      <td>Mi [vacación]{vacaciones}&lt;in:s:noun:inan&gt; perf...</td>\n",
       "      <td>[Mi vacación perfecta consiste en muchas cosas...</td>\n",
       "      <td>[Mi vacación perfecta consiste en muchas cosas...</td>\n",
       "      <td>[Mi [vacación]{vacaciones}&lt;in:s:noun:inan&gt; per...</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>174300</td>\n",
       "      <td>famous</td>\n",
       "      <td>S18</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Mi jugador favorito es Lebron James. Él es un ...</td>\n",
       "      <td>Mi jugador favorito es Lebron James. Él es un ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mi jugador favorito es Lebron James. Él es un ...</td>\n",
       "      <td>Mi jugador favorito es Lebron James. Él es un ...</td>\n",
       "      <td>512</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Mi jugador favorito es Lebron James. Él es un ...</td>\n",
       "      <td>Mi jugador favorito es Lebron James. Él es un ...</td>\n",
       "      <td>[Mi jugador favorito es Lebron James., Él es u...</td>\n",
       "      <td>[Mi jugador favorito es Lebron James., Él es u...</td>\n",
       "      <td>[Mi jugador favorito es Lebron James., Él es u...</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>145574</td>\n",
       "      <td>vacation</td>\n",
       "      <td>S18</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Las vacaciones perfectas sería para Hawai. Yo ...</td>\n",
       "      <td>Las vacaciones perfectas sería para Hawai. Yo ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Las vacaciones perfectas sería para Hawai. Yo ...</td>\n",
       "      <td>Las vacaciones perfectas sería para Hawai. Yo ...</td>\n",
       "      <td>1535</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Las vacaciones perfectas sería para Hawai. Yo ...</td>\n",
       "      <td>Las vacaciones perfectas sería para Hawai. Yo ...</td>\n",
       "      <td>[Las vacaciones perfectas sería para Hawai., Y...</td>\n",
       "      <td>[Las vacaciones perfectas sería para Hawai., Y...</td>\n",
       "      <td>[Las vacaciones perfectas sería para Hawai., Y...</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5db33e5e-e170-4794-b580-797ff7be9d28')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-5db33e5e-e170-4794-b580-797ff7be9d28 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-5db33e5e-e170-4794-b580-797ff7be9d28');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-d3869121-01e8-46a3-9692-375179719d92\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d3869121-01e8-46a3-9692-375179719d92')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-d3869121-01e8-46a3-9692-375179719d92 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "         id    prompt quarter course  \\\n",
       "163  157609    famous     S17  SPA 3   \n",
       "205  125485    famous     S17  SPA 3   \n",
       "44   111462  vacation     S17  SPA 2   \n",
       "127  174300    famous     S18  SPA 1   \n",
       "270  145574  vacation     S18  SPA 3   \n",
       "\n",
       "                                                 essay  \\\n",
       "163  \\nMi atleta favorita es Rob Gronkowski, un jug...   \n",
       "205  ----------------------------------------------...   \n",
       "44   Mi vacación perfecta consiste en muchas cosas ...   \n",
       "127  Mi jugador favorito es Lebron James. Él es un ...   \n",
       "270  Las vacaciones perfectas sería para Hawai. Yo ...   \n",
       "\n",
       "                                 a personal annotator1  \\\n",
       "163  Mi atleta favorita es Rob Gronkowski, un jugad...   \n",
       "205                                                NaN   \n",
       "44   Mi vacación perfecta consiste en muchas cosas ...   \n",
       "127  Mi jugador favorito es Lebron James. Él es un ...   \n",
       "270  Las vacaciones perfectas sería para Hawai. Yo ...   \n",
       "\n",
       "                                 a personal annotator2  \\\n",
       "163  Mi atleta favorita es Rob Gronkowski, un jugad...   \n",
       "205                                                NaN   \n",
       "44   Mi vacación perfecta consiste en muchas cosas ...   \n",
       "127                                                NaN   \n",
       "270                                                NaN   \n",
       "\n",
       "                              gender-number annotator1  \\\n",
       "163  Mi atleta [favorita]{favorito}<ga:mf:adj:aty:a...   \n",
       "205                                                NaN   \n",
       "44   Mi [vacación]{vacaciones}<in:s:noun:inan> perf...   \n",
       "127  Mi jugador favorito es Lebron James. Él es un ...   \n",
       "270  Las vacaciones perfectas sería para Hawai. Yo ...   \n",
       "\n",
       "                              gender-number annotator2  paragraph_id  ...  \\\n",
       "163  Mi atleta [favorita]{favorito}<ga:mf:adj:aty:a...           339  ...   \n",
       "205                                                NaN           381  ...   \n",
       "44   Mi [vacación]{vacaciones}<in:s:noun:inan> perf...          1119  ...   \n",
       "127  Mi jugador favorito es Lebron James. Él es un ...           512  ...   \n",
       "270  Las vacaciones perfectas sería para Hawai. Yo ...          1535  ...   \n",
       "\n",
       "     g_anno1_error_cnt  g_anno2_error_cnt  \\\n",
       "163                 17                 17   \n",
       "205                  0                  0   \n",
       "44                   4                  4   \n",
       "127                  0                  0   \n",
       "270                  3                  3   \n",
       "\n",
       "                                       p_annotator_sel  \\\n",
       "163  Mi atleta favorita es Rob Gronkowski, un jugad...   \n",
       "205                                                NaN   \n",
       "44   Mi vacación perfecta consiste en muchas cosas ...   \n",
       "127  Mi jugador favorito es Lebron James. Él es un ...   \n",
       "270  Las vacaciones perfectas sería para Hawai. Yo ...   \n",
       "\n",
       "                                       g_annotator_sel  \\\n",
       "163  Mi atleta [favorita]{favorito}<ga:mf:adj:aty:a...   \n",
       "205                                                NaN   \n",
       "44   Mi [vacación]{vacaciones}<in:s:noun:inan> perf...   \n",
       "127  Mi jugador favorito es Lebron James. Él es un ...   \n",
       "270  Las vacaciones perfectas sería para Hawai. Yo ...   \n",
       "\n",
       "                                             essay_new  \\\n",
       "163  [\\nMi atleta favorita es Rob Gronkowski, un ju...   \n",
       "205  [---------------------------------------------...   \n",
       "44   [Mi vacación perfecta consiste en muchas cosas...   \n",
       "127  [Mi jugador favorito es Lebron James., Él es u...   \n",
       "270  [Las vacaciones perfectas sería para Hawai., Y...   \n",
       "\n",
       "                                           p_annotator  \\\n",
       "163  [Mi atleta favorita es Rob Gronkowski, un juga...   \n",
       "205                                                 []   \n",
       "44   [Mi vacación perfecta consiste en muchas cosas...   \n",
       "127  [Mi jugador favorito es Lebron James., Él es u...   \n",
       "270  [Las vacaciones perfectas sería para Hawai., Y...   \n",
       "\n",
       "                                           g_annotator p_annotator_cnt  \\\n",
       "163  [Mi atleta [favorita]{favorito}<ga:mf:adj:aty:...              19   \n",
       "205                                                 []               0   \n",
       "44   [Mi [vacación]{vacaciones}<in:s:noun:inan> per...              32   \n",
       "127  [Mi jugador favorito es Lebron James., Él es u...              21   \n",
       "270  [Las vacaciones perfectas sería para Hawai., Y...              25   \n",
       "\n",
       "    g_annotator_cnt  essay_new_cnt  \n",
       "163              19             19  \n",
       "205               0             24  \n",
       "44               32             32  \n",
       "127              21             21  \n",
       "270              25             25  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8ggWCzIuO3MI"
   },
   "source": [
    "## Seperate the data need manual sorting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SE6YLFYAO3MJ"
   },
   "outputs": [],
   "source": [
    "well_formated_df = df[(df['essay_new_cnt'] == df['p_annotator_cnt']) & (df['essay_new_cnt'] == df['p_annotator_cnt'])  &(df['essay_new_cnt'] == df['g_annotator_cnt']) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TS9XaQHTO3MJ"
   },
   "outputs": [],
   "source": [
    "bad_formated_df = df[(df['essay_new_cnt'] != df['p_annotator_cnt']) | (df['essay_new_cnt'] != df['p_annotator_cnt'])  |(df['essay_new_cnt'] != df['g_annotator_cnt']) ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "105OGo0kDkmT"
   },
   "source": [
    "To do - loading the data back"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ts56hnHhsbWp"
   },
   "source": [
    "# Start Running from HERE!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "20nuLTkPrrry",
    "outputId": "433bf801-42b6-4f5d-8f18-10377dc1cfc3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "# download the bad_formated_df_edit.csv and well_formated_df.csv from share google drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eDeO9PbtDkKc"
   },
   "outputs": [],
   "source": [
    "df_well_bad_combine = pd.concat([pd.read_csv('well_formated_df.csv'),pd.read_csv('bad_formated_df_edit.csv')], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rNHx9q8cO3MJ"
   },
   "source": [
    "## Explode the sentence into row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z0gPKRtHO3MJ"
   },
   "outputs": [],
   "source": [
    "df_v1 = df_well_bad_combine[['paragraph_id', 'course','essay_new', 'p_annotator', 'g_annotator']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ohLQrddE4ICr",
    "outputId": "3f1c0c99-420d-4e54-e483-43f1eababeae"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-366f63573e9d>:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_v1['essay_new'] = df_v1['essay_new'].apply(ast.literal_eval)\n",
      "<ipython-input-5-366f63573e9d>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_v1['p_annotator'] = df_v1['p_annotator'].apply(ast.literal_eval)\n",
      "<ipython-input-5-366f63573e9d>:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_v1['g_annotator'] = df_v1['g_annotator'].apply(ast.literal_eval)\n"
     ]
    }
   ],
   "source": [
    "import ast\n",
    "#turn string back to list\n",
    "df_v1['essay_new'] = df_v1['essay_new'].apply(ast.literal_eval)\n",
    "df_v1['p_annotator'] = df_v1['p_annotator'].apply(ast.literal_eval)\n",
    "df_v1['g_annotator'] = df_v1['g_annotator'].apply(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KGKAGVdqO3MJ"
   },
   "outputs": [],
   "source": [
    "df_split = df_v1.explode(['essay_new', 'p_annotator', 'g_annotator'])\n",
    "\n",
    "# add sentence id\n",
    "df_split['sentence_id'] = df_split['essay_new'].apply(lambda row: hashlib.sha256(str(row).encode()).hexdigest())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "d4n0iKTihE50"
   },
   "outputs": [],
   "source": [
    "### remove <gat:noun:inan> <gat:nan:an> <gat:fm:noun:an> and remove [a]{a}<aa:do:an>:\n",
    "### they are indicator why the surrounding annotations are correct not an error\n",
    "\n",
    "df_split['p_annotator'] = df_split['p_annotator'].str.replace('\\[a\\]\\{a\\}\\<aa:do:an\\>', '', regex= True)\n",
    "\n",
    "df_split['g_annotator'] = df_split['g_annotator'].str.replace('\\<gat:noun:inan\\>', '', regex= True)\n",
    "df_split['g_annotator'] = df_split['g_annotator'].str.replace('\\<gat:noun:an\\>', '', regex= True)\n",
    "df_split['g_annotator'] = df_split['g_annotator'].str.replace('\\<gat:fm:noun:an\\>', '', regex= True)\n",
    "df_split['g_annotator'] = df_split['g_annotator'].str.replace('\\[a\\]\\{a\\}\\<aa:do:an\\>', '', regex= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hGssstMcO3MJ"
   },
   "outputs": [],
   "source": [
    "# # check how many errors contain in the paragraph\n",
    "# pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "\n",
    "# df_split['p_anno_error_cnt'] = df_split['p_annotator'].str.count(pattern)\n",
    "# df_split['g_anno_error_cnt'] = df_split['g_annotator'].str.count(pattern)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LLWNmafIt4TT"
   },
   "outputs": [],
   "source": [
    "# normalize the dataset\n",
    "df1 = df_split[['paragraph_id', 'sentence_id','course','essay_new', 'p_annotator']]\n",
    "df2 = df_split[['paragraph_id', 'sentence_id','course','essay_new', 'g_annotator']]\n",
    "\n",
    "df1 = df1.rename(columns={'p_annotator': 'annotated_sentence'})\n",
    "df2 = df2.rename(columns={'g_annotator': 'annotated_sentence'})\n",
    "\n",
    "\n",
    "df_normalize = pd.concat([df1, df2], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PgFsNMH2vDXP"
   },
   "outputs": [],
   "source": [
    "# check how many errors contain in the paragraph\n",
    "pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "df_normalize['anno_error_cnt'] = df_normalize['annotated_sentence'].str.count(pattern)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cOsvX78TtVNW"
   },
   "outputs": [],
   "source": [
    "#manual corrected annotation\n",
    "df_normalize.loc[(df_normalize.sentence_id == '3d1f91a13e2047b48e277796cbd7ac36e6cbb305181882e08b9979f9bdade1ff') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='[Esta]{estas}<na:ps:pron:inan> es [mi]{mis}<na:ps:det:inan> vacaciones perfectas porque me gustaría viajar a algún lugar lejos, hay animales exóticos, y muchas oportunidades para tomar fotos de personas, lugares, y animales.'\n",
    "df_normalize.loc[(df_normalize.sentence_id == 'a677e8e63ee8562ba42ed3a7e3cf5493f5a7ee622e5a7b5b70054a521bd63537') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='Muchas personas han afirmado que debido a sus diversas experiencias como un niño, [estos]{estas}<ga:fm:pron:inan> ayudaron a influir en sus escritos posteriores que se hizo muy [popular]{populares}<na:ps:adj:inan>.'\n",
    "df_normalize.loc[(df_normalize.sentence_id == 'd822cea1fd4f64f0a2799221ef8bc59a4056a0b5aed64aaa9f5fb540aa683954') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='Y tenìa éxito porque después sus esfuerzos, trabajadores en Texas, California, Florida, y Arizona tenìan condicionales mejores, ganaron más dinero, y tenían más [derechas]{derechos}<ig:f:noun:inan>.'\n",
    "df_normalize.loc[(df_normalize.sentence_id == 'bde831835adaf8066ae64bee72fc5ba6005fb45670308886d171246f3f2a9498') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='Pero mi [vacacion]{vacaciones}<in:s:noun:inan> [perfecto]{perfecta}<ga:fm:adj:inan> es en Costa Rica.'\n",
    "df_normalize.loc[(df_normalize.sentence_id == 'ecf3c1e7ff36b596e6ac73e85947277327a75348c7c2cd3cae6fde858df3aa55') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='Debido de la llueve las plantas son muy [verde]{verdes}<na:ps:adj:an> y bonitas'\n",
    "df_normalize.loc[(df_normalize.sentence_id == 'e89e43da8c69782b0fd148b68c96380b35b0edb867f15b3c62c25305007f9faf') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='Biles fue tres veces [campeón]{campeona}<ga:fm:noun:an> mundial, tres veces [campeón]{campeona}<ga:fm:noun:an> mundial, dos veces [campeón]{campeona}<ga:fm:noun:an> mundial de balanza de balanza, cuatro veces [campeón]{campeona}<ga:fm:noun:an> nacional de Estados Unidos y miembro de los equipos estadounidenses ganadores de medallas de oro en los Campeonatos Mundiales de Gimnasia Artística 2014 y 2015.'\n",
    "df_normalize.loc[(df_normalize.sentence_id == 'c302a46b78dd40fc7f27afc89099d89cb510e58a1b5920d9c6cafaec871aa131') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='México es [una]{un}<ga:mf:det:inan> país [hermosa]{hermoso}<ga:mf:adj:inan> y lleno de historia.'\n",
    "df_normalize.loc[(df_normalize.sentence_id == '71ce36921a0034880de8a18a578bd756e9be78eedd2d52fcd9d58cd9b482be7e') & (df_normalize.anno_error_cnt !=0),'annotated_sentence']='Quiero que mis hermanos y primos vengan, también, porque ellos son muy [divertidas]{divertidos}<ga:mf:adj:an> y [entretenidas]{entretenidos}<ga:mf:adj:an> y haría la [vacación]{vacaciones}<in:s:noun:inan> mucho más [divertidas]{divertidos}<ga:mf:adj:an>.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nXnzz6f1wsA_"
   },
   "outputs": [],
   "source": [
    "# df_normalize.loc[(df_normalize.sentence_id == '71ce36921a0034880de8a18a578bd756e9be78eedd2d52fcd9d58cd9b482be7e')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iKPS-Kr_O3MJ"
   },
   "outputs": [],
   "source": [
    "# extracting all the errors into the list\n",
    "df_normalize['full_error_tag'] = df_normalize['annotated_sentence'].str.findall('\\[.*?\\]\\{.*?\\}\\<.*?\\>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "52QGzLAmvXk_"
   },
   "outputs": [],
   "source": [
    "#replace '\\xa0' to space\n",
    "#replace \"\" to \"\n",
    "#replace \\n\\n to space\n",
    "\n",
    "df_normalize['annotated_sentence'] = df_normalize['annotated_sentence'].str.replace('\\xa0', ' ', regex= True)\n",
    "\n",
    "df_normalize['annotated_sentence'] = df_normalize['annotated_sentence'].str.replace('\\\"\\\"', '\\\"', regex= True)\n",
    "\n",
    "df_normalize['annotated_sentence'] = df_normalize['annotated_sentence'].str.replace('\\n\\n', ' ', regex= True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nd4poECEgKQY"
   },
   "outputs": [],
   "source": [
    "def create_order_tag(list):\n",
    "  return [i for i in range(len(list))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "70_hKOI8jmF_"
   },
   "outputs": [],
   "source": [
    "# add order tag for later processing\n",
    "df_normalize['order_tag'] = df_normalize['full_error_tag'].apply(create_order_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "f5Y4rb1nO3MJ"
   },
   "outputs": [],
   "source": [
    "# sentence with multiple error tags get explode into copy of the corresponding sentence\n",
    "df_final = df_normalize.explode(['full_error_tag', 'order_tag'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uAskn4w5FJoC"
   },
   "outputs": [],
   "source": [
    "# get error_tag  #correction  #error\n",
    "\n",
    "df_final['error_tag'] = df_final['full_error_tag'].str.extract(r'\\[.*?\\]\\{.*?\\}\\<(.*?)\\>')\n",
    "df_final['correction'] = df_final['full_error_tag'].str.extract(r'\\[.*?\\]\\{(.*?)\\}\\<.*?\\>')\n",
    "df_final['error'] = df_final['full_error_tag'].str.extract(r'\\[(.*?)\\]\\{.*?\\}\\<.*?\\>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 597
    },
    "id": "GtsDKfiQO3MK",
    "outputId": "f348b6b4-e9f5-4f34-b980-f643b0359b9e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-c970c191-5186-4b53-8795-da9bf2e42bbf\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>anno_error_cnt</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>order_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2...</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanil...</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a579...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ...</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18</td>\n",
       "      <td>66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>2</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18</td>\n",
       "      <td>66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>2</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>28</td>\n",
       "      <td>0e02104bc91f910260846ddba52137596f5b83525260c1...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden ...</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1596</td>\n",
       "      <td>925b7b7adc4c6be7c5cd4f33f80d8a24156380c9790b23...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>La perfecta vacación para mi seria ir al Caribe.</td>\n",
       "      <td>La perfecta [vacación]{vacaciones}&lt;in:s:noun:i...</td>\n",
       "      <td>1</td>\n",
       "      <td>[vacación]{vacaciones}&lt;in:s:noun:inan&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>in:s:noun:inan</td>\n",
       "      <td>vacaciones</td>\n",
       "      <td>vacación</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1596</td>\n",
       "      <td>7b6c6008e1c0c9cde850383d4c3bc738bb0df407526a8e...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Yo estado soñando con esta vacaciones desde qu...</td>\n",
       "      <td>Yo estado soñando con [esta]{estas}&lt;na:ps:det:...</td>\n",
       "      <td>1</td>\n",
       "      <td>[esta]{estas}&lt;na:ps:det:inan&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>na:ps:det:inan</td>\n",
       "      <td>estas</td>\n",
       "      <td>esta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1623</td>\n",
       "      <td>6b6c17a8ce70e6f2754008b0329ae4a85622594d878052...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Unas vacaciones perfectas es París porque eran...</td>\n",
       "      <td>Unas vacaciones perfectas es París porque eran...</td>\n",
       "      <td>1</td>\n",
       "      <td>[muchos]{muchas}&lt;ga:fm:det:inan&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>ga:fm:det:inan</td>\n",
       "      <td>muchas</td>\n",
       "      <td>muchos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1623</td>\n",
       "      <td>d136aa5a6cdd9ba0c56a12b3e4b24b37f0f8293d013291...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Entonces será certificado como “perfecto\".</td>\n",
       "      <td>Entonces será [certificado]{certificadas}&lt;ga:f...</td>\n",
       "      <td>2</td>\n",
       "      <td>[certificado]{certificadas}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>ga:fm:adj:inan</td>\n",
       "      <td>certificadas</td>\n",
       "      <td>certificado</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1623</td>\n",
       "      <td>d136aa5a6cdd9ba0c56a12b3e4b24b37f0f8293d013291...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Entonces será certificado como “perfecto\".</td>\n",
       "      <td>Entonces será [certificado]{certificadas}&lt;ga:f...</td>\n",
       "      <td>2</td>\n",
       "      <td>[perfecto]{perfectas}&lt;ga:na:fm:ps:adj:inan&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>ga:na:fm:ps:adj:inan</td>\n",
       "      <td>perfectas</td>\n",
       "      <td>perfecto</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5223 rows × 11 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c970c191-5186-4b53-8795-da9bf2e42bbf')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-c970c191-5186-4b53-8795-da9bf2e42bbf button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-c970c191-5186-4b53-8795-da9bf2e42bbf');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-f00cb662-604a-4341-a4bd-9cec887ac7dc\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f00cb662-604a-4341-a4bd-9cec887ac7dc')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-f00cb662-604a-4341-a4bd-9cec887ac7dc button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id                                        sentence_id course  \\\n",
       "6             13  452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2...  SPA 1   \n",
       "9             18  e627490f6722ef9caeb07e150107b31bcc9c088016a579...  SPA 2   \n",
       "9             18  66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...  SPA 2   \n",
       "9             18  66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...  SPA 2   \n",
       "17            28  0e02104bc91f910260846ddba52137596f5b83525260c1...  SPA 3   \n",
       "..           ...                                                ...    ...   \n",
       "46          1596  925b7b7adc4c6be7c5cd4f33f80d8a24156380c9790b23...  SPA 3   \n",
       "46          1596  7b6c6008e1c0c9cde850383d4c3bc738bb0df407526a8e...  SPA 3   \n",
       "47          1623  6b6c17a8ce70e6f2754008b0329ae4a85622594d878052...  SPA 2   \n",
       "47          1623  d136aa5a6cdd9ba0c56a12b3e4b24b37f0f8293d013291...  SPA 2   \n",
       "47          1623  d136aa5a6cdd9ba0c56a12b3e4b24b37f0f8293d013291...  SPA 2   \n",
       "\n",
       "                                            essay_new  \\\n",
       "6   No confundas Selena Gomez con Selena Quintanil...   \n",
       "9   Un Otro famoso actor, que llama Nicolas Cage, ...   \n",
       "9   No tiene muchos compañeros que le gusta y ento...   \n",
       "9   No tiene muchos compañeros que le gusta y ento...   \n",
       "17  Princesa Anna y sus amigas quieren que ayuden ...   \n",
       "..                                                ...   \n",
       "46   La perfecta vacación para mi seria ir al Caribe.   \n",
       "46  Yo estado soñando con esta vacaciones desde qu...   \n",
       "47  Unas vacaciones perfectas es París porque eran...   \n",
       "47         Entonces será certificado como “perfecto\".   \n",
       "47         Entonces será certificado como “perfecto\".   \n",
       "\n",
       "                                   annotated_sentence  anno_error_cnt  \\\n",
       "6   No confundas []{a}<az:do:an> Selena Gomez con ...               1   \n",
       "9   Un Otro famoso actor, que llama Nicolas Cage, ...               1   \n",
       "9   No tiene muchos compañeros que le gusta y ento...               2   \n",
       "9   No tiene muchos compañeros que le gusta y ento...               2   \n",
       "17  Princesa Anna y sus amigas quieren que ayuden ...               1   \n",
       "..                                                ...             ...   \n",
       "46  La perfecta [vacación]{vacaciones}<in:s:noun:i...               1   \n",
       "46  Yo estado soñando con [esta]{estas}<na:ps:det:...               1   \n",
       "47  Unas vacaciones perfectas es París porque eran...               1   \n",
       "47  Entonces será [certificado]{certificadas}<ga:f...               2   \n",
       "47  Entonces será [certificado]{certificadas}<ga:f...               2   \n",
       "\n",
       "                                 full_error_tag order_tag  \\\n",
       "6                               []{a}<az:do:an>         0   \n",
       "9                               []{a}<az:do:an>         0   \n",
       "9                               []{a}<az:do:an>         0   \n",
       "9                               []{a}<az:do:an>         1   \n",
       "17                              []{a}<az:do:an>         0   \n",
       "..                                          ...       ...   \n",
       "46       [vacación]{vacaciones}<in:s:noun:inan>         0   \n",
       "46                [esta]{estas}<na:ps:det:inan>         0   \n",
       "47             [muchos]{muchas}<ga:fm:det:inan>         0   \n",
       "47  [certificado]{certificadas}<ga:fm:adj:inan>         0   \n",
       "47  [perfecto]{perfectas}<ga:na:fm:ps:adj:inan>         1   \n",
       "\n",
       "               error_tag    correction        error  \n",
       "6               az:do:an             a               \n",
       "9               az:do:an             a               \n",
       "9               az:do:an             a               \n",
       "9               az:do:an             a               \n",
       "17              az:do:an             a               \n",
       "..                   ...           ...          ...  \n",
       "46        in:s:noun:inan    vacaciones     vacación  \n",
       "46        na:ps:det:inan         estas         esta  \n",
       "47        ga:fm:det:inan        muchas       muchos  \n",
       "47        ga:fm:adj:inan  certificadas  certificado  \n",
       "47  ga:na:fm:ps:adj:inan     perfectas     perfecto  \n",
       "\n",
       "[5223 rows x 11 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[df_final['error_tag'].notnull()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Kt_3xBfLfVPI"
   },
   "source": [
    "## Feature Engineering data for GEC model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Tkc7tlMGYqTQ"
   },
   "source": [
    "### for sentence with no error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PYuRxCuLUZmj"
   },
   "outputs": [],
   "source": [
    "# df_final_without_error = df_final[df_final['error_tag'].isnull()]\n",
    "# df_final_without_error['error_type'] = 'None'\n",
    "# df_final_without_error['correction_index'] = [[] for i in range(len(df_final_without_error))]\n",
    "# df_final_without_error['source'] = df_final_without_error['essay_new'].apply(lambda x: x.strip()).apply(nltk.word_tokenize)\n",
    "# df_final_without_error['target'] = df_final_without_error['annotated_sentence'].apply(lambda x: x.strip()).apply(nltk.word_tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xdf6ENPkaF6a"
   },
   "outputs": [],
   "source": [
    "# df_final_without_error[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Xd9bV4j1Yxav"
   },
   "source": [
    "### for sentence with one error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UCVkwm-5YwX0"
   },
   "outputs": [],
   "source": [
    "pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "\n",
    "df_final['anno_error_cnt'] = df_final['annotated_sentence'].str.count(pattern)\n",
    "# filter out df with only one error\n",
    "df_final_with1_error = df_final[df_final['anno_error_cnt'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rygj_Rwt0AQa"
   },
   "outputs": [],
   "source": [
    "def get_error_position(sentence):\n",
    "  pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "  for i in range(len(sentence)):\n",
    "    if re.search(pattern, sentence[i]):\n",
    "      return [i, len(sentence)+1+i]\n",
    "\n",
    "def format_target(sentence):\n",
    "  pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "  for i in range(len(sentence)):\n",
    "    if re.search(pattern, sentence[i]):\n",
    "      replace_word = re.match(r'\\[.*?\\]\\{(.*?)\\}\\<.*?\\>', sentence[i]).group(1)\n",
    "      if len(replace_word) == 0:\n",
    "        sentence[i] = '[NONE]'\n",
    "      else:\n",
    "        sentence[i] = replace_word\n",
    "  return sentence\n",
    "\n",
    "def format_source(sentence):\n",
    "  pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "  for i in range(len(sentence)):\n",
    "    if re.search(pattern, sentence[i]):\n",
    "      replace_word = re.match(r'\\[(.*?)\\]\\{.*?\\}\\<.*?\\>', sentence[i]).group(1)\n",
    "      if len(replace_word) == 0:\n",
    "        sentence[i] = '[NONE]'\n",
    "      else:\n",
    "        sentence[i] = replace_word\n",
    "  return sentence\n",
    "\n",
    "def get_error_type(error_code):\n",
    "  matches = re.findall(':', error_code)\n",
    "  match_count = len(matches)\n",
    "  error_split = error_code.split(':')\n",
    "  if match_count <= 3:\n",
    "    return ':'.join(error_split[:2])\n",
    "  else:\n",
    "    return ':'.join(error_split[:4])\n",
    "\n",
    "# a = 'ga:mf:det:an'\n",
    "# b = 'aa:do:an'\n",
    "# c = 'in:s:noun:inan'\n",
    "# d = 'ga:na:fm:ps:adj:inan'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uJatibtoyVyT",
    "outputId": "7767cdd9-efa0-411a-ac2d-9255846f4ced"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-24-39efdc822ffd>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with1_error['target'] = df_final_with1_error['annotated_sentence'].apply(lambda x:re.findall(r'\\b\\w+\\b|\\[.*?\\]\\{.*?\\}\\<.*?\\>|\\S', x))\n",
      "<ipython-input-24-39efdc822ffd>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with1_error['correction_index'] = df_final_with1_error['target'].apply(get_error_position)\n",
      "<ipython-input-24-39efdc822ffd>:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with1_error['error_type'] = df_final_with1_error['error_tag'].apply(get_error_type)\n",
      "<ipython-input-24-39efdc822ffd>:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with1_error['target'] = df_final_with1_error['target'].apply(format_target)\n",
      "<ipython-input-24-39efdc822ffd>:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with1_error['source'] = df_final_with1_error['annotated_sentence'].apply(lambda x:re.findall(r'\\b\\w+\\b|\\[.*?\\]\\{.*?\\}\\<.*?\\>|\\S', x))\n",
      "<ipython-input-24-39efdc822ffd>:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with1_error['source'] = df_final_with1_error['source'].apply(format_source)\n"
     ]
    }
   ],
   "source": [
    "# turn annotated sentence into a list\n",
    "df_final_with1_error['target'] = df_final_with1_error['annotated_sentence'].apply(lambda x:re.findall(r'\\b\\w+\\b|\\[.*?\\]\\{.*?\\}\\<.*?\\>|\\S', x))\n",
    "# get the error position\n",
    "df_final_with1_error['correction_index'] = df_final_with1_error['target'].apply(get_error_position)\n",
    "df_final_with1_error['error_type'] = df_final_with1_error['error_tag'].apply(get_error_type)\n",
    "df_final_with1_error['target'] = df_final_with1_error['target'].apply(format_target)\n",
    "df_final_with1_error['source'] = df_final_with1_error['annotated_sentence'].apply(lambda x:re.findall(r'\\b\\w+\\b|\\[.*?\\]\\{.*?\\}\\<.*?\\>|\\S', x))\n",
    "df_final_with1_error['source'] = df_final_with1_error['source'].apply(format_source)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 660
    },
    "id": "RD_7xS3yCte0",
    "outputId": "8b6ee8c2-f996-45d0-c186-5aa8f42a88a4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-b3d1b23b-3377-45aa-bf93-f5d95a1e9bf8\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>anno_error_cnt</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>order_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>error_type</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2...</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanil...</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[No, confundas, a, Selena, Gomez, con, Selena,...</td>\n",
       "      <td>[2, 19]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[No, confundas, [NONE], Selena, Gomez, con, Se...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a579...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ...</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicol...</td>\n",
       "      <td>[11, 32]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>28</td>\n",
       "      <td>0e02104bc91f910260846ddba52137596f5b83525260c1...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden ...</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Princesa, Anna, y, sus, amigas, quieren, que,...</td>\n",
       "      <td>[8, 21]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Princesa, Anna, y, sus, amigas, quieren, que,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>28</td>\n",
       "      <td>dda3163fe647d016c4f212fd794343bd1b2446243a6ac3...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Pero Eleanor dice su secreto a su amigo Chiddi...</td>\n",
       "      <td>Pero Eleanor dice su secreto a su amigo Chiddi...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Pero, Eleanor, dice, su, secreto, a, su, amig...</td>\n",
       "      <td>[13, 30]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Pero, Eleanor, dice, su, secreto, a, su, amig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>28</td>\n",
       "      <td>abb0ff0f493ec9cabdf7eaeb7b0e0078879fbe59f5768f...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Chiddi enseña Eleanor sobre cómo ser una perso...</td>\n",
       "      <td>Chiddi enseña []{a}&lt;az:do:an&gt; Eleanor sobre có...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Chiddi, enseña, a, Eleanor, sobre, cómo, ser,...</td>\n",
       "      <td>[2, 14]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Chiddi, enseña, [NONE], Eleanor, sobre, cómo,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b3d1b23b-3377-45aa-bf93-f5d95a1e9bf8')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-b3d1b23b-3377-45aa-bf93-f5d95a1e9bf8 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-b3d1b23b-3377-45aa-bf93-f5d95a1e9bf8');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-418798b2-7017-44d4-9e6e-6f514e032137\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-418798b2-7017-44d4-9e6e-6f514e032137')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-418798b2-7017-44d4-9e6e-6f514e032137 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id                                        sentence_id course  \\\n",
       "6             13  452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2...  SPA 1   \n",
       "9             18  e627490f6722ef9caeb07e150107b31bcc9c088016a579...  SPA 2   \n",
       "17            28  0e02104bc91f910260846ddba52137596f5b83525260c1...  SPA 3   \n",
       "17            28  dda3163fe647d016c4f212fd794343bd1b2446243a6ac3...  SPA 3   \n",
       "17            28  abb0ff0f493ec9cabdf7eaeb7b0e0078879fbe59f5768f...  SPA 3   \n",
       "\n",
       "                                            essay_new  \\\n",
       "6   No confundas Selena Gomez con Selena Quintanil...   \n",
       "9   Un Otro famoso actor, que llama Nicolas Cage, ...   \n",
       "17  Princesa Anna y sus amigas quieren que ayuden ...   \n",
       "17  Pero Eleanor dice su secreto a su amigo Chiddi...   \n",
       "17  Chiddi enseña Eleanor sobre cómo ser una perso...   \n",
       "\n",
       "                                   annotated_sentence  anno_error_cnt  \\\n",
       "6   No confundas []{a}<az:do:an> Selena Gomez con ...               1   \n",
       "9   Un Otro famoso actor, que llama Nicolas Cage, ...               1   \n",
       "17  Princesa Anna y sus amigas quieren que ayuden ...               1   \n",
       "17  Pero Eleanor dice su secreto a su amigo Chiddi...               1   \n",
       "17  Chiddi enseña []{a}<az:do:an> Eleanor sobre có...               1   \n",
       "\n",
       "     full_error_tag order_tag error_tag correction error  \\\n",
       "6   []{a}<az:do:an>         0  az:do:an          a         \n",
       "9   []{a}<az:do:an>         0  az:do:an          a         \n",
       "17  []{a}<az:do:an>         0  az:do:an          a         \n",
       "17  []{a}<az:do:an>         0  az:do:an          a         \n",
       "17  []{a}<az:do:an>         0  az:do:an          a         \n",
       "\n",
       "                                               target correction_index  \\\n",
       "6   [No, confundas, a, Selena, Gomez, con, Selena,...          [2, 19]   \n",
       "9   [Un, Otro, famoso, actor, ,, que, llama, Nicol...         [11, 32]   \n",
       "17  [Princesa, Anna, y, sus, amigas, quieren, que,...          [8, 21]   \n",
       "17  [Pero, Eleanor, dice, su, secreto, a, su, amig...         [13, 30]   \n",
       "17  [Chiddi, enseña, a, Eleanor, sobre, cómo, ser,...          [2, 14]   \n",
       "\n",
       "   error_type                                             source  \n",
       "6       az:do  [No, confundas, [NONE], Selena, Gomez, con, Se...  \n",
       "9       az:do  [Un, Otro, famoso, actor, ,, que, llama, Nicol...  \n",
       "17      az:do  [Princesa, Anna, y, sus, amigas, quieren, que,...  \n",
       "17      az:do  [Pero, Eleanor, dice, su, secreto, a, su, amig...  \n",
       "17      az:do  [Chiddi, enseña, [NONE], Eleanor, sobre, cómo,...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_with1_error[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "idh20q5iY1DN"
   },
   "source": [
    "### for sentence with 2+ errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tKJevuA5Hvh1"
   },
   "outputs": [],
   "source": [
    "# filter out df with only two plus errors\n",
    "df_final_with2_error = df_final[df_final['anno_error_cnt'] > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QGekzoZadFq1"
   },
   "outputs": [],
   "source": [
    "def get_error_position_multi(sentence, order_tag):\n",
    "   formated_sentence = re.findall(r'\\b\\w+\\b|\\[.*?\\]\\{.*?\\}\\<.*?\\>|\\S', sentence)\n",
    "   pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "   current = 0\n",
    "   for i in range(len(formated_sentence)):\n",
    "      if re.search(pattern, formated_sentence[i]):\n",
    "        if current == int(order_tag):\n",
    "          return [i, len(formated_sentence)+1+i]\n",
    "        else:\n",
    "          current += 1\n",
    "\n",
    "# print(get_error_position_multi( 'tiene []{a}<az:do:an> muchos compañeros []{a}<az:do:an>', '1'))\n",
    "\n",
    "def format_source_multi(sentence, order_tag):\n",
    "  formated_sentence = re.findall(r'\\b\\w+\\b|\\[.*?\\]\\{.*?\\}\\<.*?\\>|\\S', sentence)\n",
    "  pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "  current = 0\n",
    "  for i in range(len(formated_sentence)):\n",
    "    if re.search(pattern, formated_sentence[i]):\n",
    "      if current == int(order_tag):\n",
    "        current += 1\n",
    "        replace_word = re.match(r'\\[(.*?)\\]\\{.*?\\}\\<.*?\\>', formated_sentence[i]).group(1)\n",
    "      else:\n",
    "        current += 1\n",
    "        replace_word = re.match(r'\\[.*?\\]\\{(.*?)\\}\\<.*?\\>', formated_sentence[i]).group(1)\n",
    "      if len(replace_word) == 0:\n",
    "        formated_sentence[i] = '[NONE]'\n",
    "      else:\n",
    "        formated_sentence[i] = replace_word\n",
    "  return formated_sentence\n",
    "\n",
    "# print(format_source_multi( 'tiene []{a}<az:do:an> muchos compañeros []{a}<az:do:an>', '1'))\n",
    "\n",
    "\n",
    "def format_target_multi(sentence, order_tag):\n",
    "  formated_sentence = re.findall(r'\\b\\w+\\b|\\[.*?\\]\\{.*?\\}\\<.*?\\>|\\S', sentence)\n",
    "  pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "  current = 0\n",
    "  for i in range(len(formated_sentence)):\n",
    "    if re.search(pattern, formated_sentence[i]):\n",
    "      replace_word = re.match(r'\\[.*?\\]\\{(.*?)\\}\\<.*?\\>', formated_sentence[i]).group(1)\n",
    "      if len(replace_word) == 0:\n",
    "        formated_sentence[i] = '[NONE]'\n",
    "      else:\n",
    "        formated_sentence[i] = replace_word\n",
    "  return formated_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "13NI130BrKnC",
    "outputId": "a80e8486-fc54-49a7-c6a3-e20bb7b9bb1e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-28-3158e6a61d53>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with2_error['correction_index'] = df_final_with2_error.apply(lambda row : get_error_position_multi(row['annotated_sentence'], row['order_tag']), axis=1)\n",
      "<ipython-input-28-3158e6a61d53>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with2_error['error_type'] = df_final_with2_error['error_tag'].apply(get_error_type)\n",
      "<ipython-input-28-3158e6a61d53>:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with2_error['source'] = df_final_with2_error.apply(lambda row : format_source_multi(row['annotated_sentence'], row['order_tag']), axis=1)\n",
      "<ipython-input-28-3158e6a61d53>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final_with2_error['target'] = df_final_with2_error.apply(lambda row : format_target_multi(row['annotated_sentence'], row['order_tag']), axis=1)\n"
     ]
    }
   ],
   "source": [
    "df_final_with2_error['correction_index'] = df_final_with2_error.apply(lambda row : get_error_position_multi(row['annotated_sentence'], row['order_tag']), axis=1)\n",
    "df_final_with2_error['error_type'] = df_final_with2_error['error_tag'].apply(get_error_type)\n",
    "df_final_with2_error['source'] = df_final_with2_error.apply(lambda row : format_source_multi(row['annotated_sentence'], row['order_tag']), axis=1)\n",
    "df_final_with2_error['target'] = df_final_with2_error.apply(lambda row : format_target_multi(row['annotated_sentence'], row['order_tag']), axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 591
    },
    "id": "oAg6jsP-cXgx",
    "outputId": "3883ef5f-4fe3-4084-ef02-2e1a3ecd8ba9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-335b8125-b5a9-42b9-962b-ff4c95b507e7\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>anno_error_cnt</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>order_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>error_type</th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18</td>\n",
       "      <td>66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>2</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[11, 40]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[No, tiene, muchos, compañeros, que, le, gusta...</td>\n",
       "      <td>[No, tiene, muchos, compañeros, que, le, gusta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18</td>\n",
       "      <td>66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>No tiene muchos compañeros que le gusta y ento...</td>\n",
       "      <td>2</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[15, 44]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[No, tiene, muchos, compañeros, que, le, gusta...</td>\n",
       "      <td>[No, tiene, muchos, compañeros, que, le, gusta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>29</td>\n",
       "      <td>26d89d179a3eb5c85bf5348fe731337964665da37843a9...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Y el fui a mata personas porque ellos mataron ...</td>\n",
       "      <td>Y el fui a mata []{a}&lt;az:do:an&gt; personas porqu...</td>\n",
       "      <td>2</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[5, 20]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Y, el, fui, a, mata, [NONE], personas, porque...</td>\n",
       "      <td>[Y, el, fui, a, mata, a, personas, porque, ell...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>29</td>\n",
       "      <td>26d89d179a3eb5c85bf5348fe731337964665da37843a9...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Y el fui a mata personas porque ellos mataron ...</td>\n",
       "      <td>Y el fui a mata []{a}&lt;az:do:an&gt; personas porqu...</td>\n",
       "      <td>2</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[10, 25]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Y, el, fui, a, mata, a, personas, porque, ell...</td>\n",
       "      <td>[Y, el, fui, a, mata, a, personas, porque, ell...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>81</td>\n",
       "      <td>53a2bc2eb7b79482760f3fe8ec2d9ee8b34da84af57349...</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Sin embargo, mucho gente estan los fanes de Wi...</td>\n",
       "      <td>Sin embargo, mucho gente estan los fanes de Wi...</td>\n",
       "      <td>2</td>\n",
       "      <td>[]{a}&lt;az:io:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:io:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[14, 45]</td>\n",
       "      <td>az:io</td>\n",
       "      <td>[Sin, embargo, ,, mucho, gente, estan, los, fa...</td>\n",
       "      <td>[Sin, embargo, ,, mucho, gente, estan, los, fa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-335b8125-b5a9-42b9-962b-ff4c95b507e7')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-335b8125-b5a9-42b9-962b-ff4c95b507e7 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-335b8125-b5a9-42b9-962b-ff4c95b507e7');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-c40de690-4322-42e9-9068-42cca2760bcb\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c40de690-4322-42e9-9068-42cca2760bcb')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-c40de690-4322-42e9-9068-42cca2760bcb button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id                                        sentence_id course  \\\n",
       "9             18  66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...  SPA 2   \n",
       "9             18  66d1b86a207371c43e54706cc1d2bfdb9a29e7d5da6c4c...  SPA 2   \n",
       "18            29  26d89d179a3eb5c85bf5348fe731337964665da37843a9...  SPA 3   \n",
       "18            29  26d89d179a3eb5c85bf5348fe731337964665da37843a9...  SPA 3   \n",
       "51            81  53a2bc2eb7b79482760f3fe8ec2d9ee8b34da84af57349...  SPA 1   \n",
       "\n",
       "                                            essay_new  \\\n",
       "9   No tiene muchos compañeros que le gusta y ento...   \n",
       "9   No tiene muchos compañeros que le gusta y ento...   \n",
       "18  Y el fui a mata personas porque ellos mataron ...   \n",
       "18  Y el fui a mata personas porque ellos mataron ...   \n",
       "51  Sin embargo, mucho gente estan los fanes de Wi...   \n",
       "\n",
       "                                   annotated_sentence  anno_error_cnt  \\\n",
       "9   No tiene muchos compañeros que le gusta y ento...               2   \n",
       "9   No tiene muchos compañeros que le gusta y ento...               2   \n",
       "18  Y el fui a mata []{a}<az:do:an> personas porqu...               2   \n",
       "18  Y el fui a mata []{a}<az:do:an> personas porqu...               2   \n",
       "51  Sin embargo, mucho gente estan los fanes de Wi...               2   \n",
       "\n",
       "     full_error_tag order_tag error_tag correction error correction_index  \\\n",
       "9   []{a}<az:do:an>         0  az:do:an          a               [11, 40]   \n",
       "9   []{a}<az:do:an>         1  az:do:an          a               [15, 44]   \n",
       "18  []{a}<az:do:an>         0  az:do:an          a                [5, 20]   \n",
       "18  []{a}<az:do:an>         1  az:do:an          a               [10, 25]   \n",
       "51  []{a}<az:io:an>         0  az:io:an          a               [14, 45]   \n",
       "\n",
       "   error_type                                             source  \\\n",
       "9       az:do  [No, tiene, muchos, compañeros, que, le, gusta...   \n",
       "9       az:do  [No, tiene, muchos, compañeros, que, le, gusta...   \n",
       "18      az:do  [Y, el, fui, a, mata, [NONE], personas, porque...   \n",
       "18      az:do  [Y, el, fui, a, mata, a, personas, porque, ell...   \n",
       "51      az:io  [Sin, embargo, ,, mucho, gente, estan, los, fa...   \n",
       "\n",
       "                                               target  \n",
       "9   [No, tiene, muchos, compañeros, que, le, gusta...  \n",
       "9   [No, tiene, muchos, compañeros, que, le, gusta...  \n",
       "18  [Y, el, fui, a, mata, a, personas, porque, ell...  \n",
       "18  [Y, el, fui, a, mata, a, personas, porque, ell...  \n",
       "51  [Sin, embargo, ,, mucho, gente, estan, los, fa...  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_with2_error[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "heX-71F6dvpn"
   },
   "source": [
    "### union and get the final dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "P2ymprM2zwUP"
   },
   "outputs": [],
   "source": [
    "# df_combine = pd.concat([df_final_without_error, df_final_with1_error, df_final_with2_error], axis=0)\n",
    "df_combine = pd.concat([df_final_with1_error, df_final_with2_error], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7uLF2RLseV5T"
   },
   "outputs": [],
   "source": [
    "df_combine.to_csv('output.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 660
    },
    "id": "6Zwagm210Tvf",
    "outputId": "2fd6d79d-c9ed-49fe-f0a6-d0a2474d4aa0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-1a929a5f-086a-4df4-8ca2-bed2886d69a6\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>anno_error_cnt</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>order_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>error_type</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2...</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanil...</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[No, confundas, a, Selena, Gomez, con, Selena,...</td>\n",
       "      <td>[2, 19]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[No, confundas, [NONE], Selena, Gomez, con, Se...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a579...</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ...</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicol...</td>\n",
       "      <td>[11, 32]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>28</td>\n",
       "      <td>0e02104bc91f910260846ddba52137596f5b83525260c1...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden ...</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Princesa, Anna, y, sus, amigas, quieren, que,...</td>\n",
       "      <td>[8, 21]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Princesa, Anna, y, sus, amigas, quieren, que,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>28</td>\n",
       "      <td>dda3163fe647d016c4f212fd794343bd1b2446243a6ac3...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Pero Eleanor dice su secreto a su amigo Chiddi...</td>\n",
       "      <td>Pero Eleanor dice su secreto a su amigo Chiddi...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Pero, Eleanor, dice, su, secreto, a, su, amig...</td>\n",
       "      <td>[13, 30]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Pero, Eleanor, dice, su, secreto, a, su, amig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>28</td>\n",
       "      <td>abb0ff0f493ec9cabdf7eaeb7b0e0078879fbe59f5768f...</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Chiddi enseña Eleanor sobre cómo ser una perso...</td>\n",
       "      <td>Chiddi enseña []{a}&lt;az:do:an&gt; Eleanor sobre có...</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "      <td>[Chiddi, enseña, a, Eleanor, sobre, cómo, ser,...</td>\n",
       "      <td>[2, 14]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>[Chiddi, enseña, [NONE], Eleanor, sobre, cómo,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1a929a5f-086a-4df4-8ca2-bed2886d69a6')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-1a929a5f-086a-4df4-8ca2-bed2886d69a6 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-1a929a5f-086a-4df4-8ca2-bed2886d69a6');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-c0e15784-d245-463d-9146-fd4a6d63a217\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c0e15784-d245-463d-9146-fd4a6d63a217')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-c0e15784-d245-463d-9146-fd4a6d63a217 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id                                        sentence_id course  \\\n",
       "6             13  452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2...  SPA 1   \n",
       "9             18  e627490f6722ef9caeb07e150107b31bcc9c088016a579...  SPA 2   \n",
       "17            28  0e02104bc91f910260846ddba52137596f5b83525260c1...  SPA 3   \n",
       "17            28  dda3163fe647d016c4f212fd794343bd1b2446243a6ac3...  SPA 3   \n",
       "17            28  abb0ff0f493ec9cabdf7eaeb7b0e0078879fbe59f5768f...  SPA 3   \n",
       "\n",
       "                                            essay_new  \\\n",
       "6   No confundas Selena Gomez con Selena Quintanil...   \n",
       "9   Un Otro famoso actor, que llama Nicolas Cage, ...   \n",
       "17  Princesa Anna y sus amigas quieren que ayuden ...   \n",
       "17  Pero Eleanor dice su secreto a su amigo Chiddi...   \n",
       "17  Chiddi enseña Eleanor sobre cómo ser una perso...   \n",
       "\n",
       "                                   annotated_sentence  anno_error_cnt  \\\n",
       "6   No confundas []{a}<az:do:an> Selena Gomez con ...               1   \n",
       "9   Un Otro famoso actor, que llama Nicolas Cage, ...               1   \n",
       "17  Princesa Anna y sus amigas quieren que ayuden ...               1   \n",
       "17  Pero Eleanor dice su secreto a su amigo Chiddi...               1   \n",
       "17  Chiddi enseña []{a}<az:do:an> Eleanor sobre có...               1   \n",
       "\n",
       "     full_error_tag order_tag error_tag correction error  \\\n",
       "6   []{a}<az:do:an>         0  az:do:an          a         \n",
       "9   []{a}<az:do:an>         0  az:do:an          a         \n",
       "17  []{a}<az:do:an>         0  az:do:an          a         \n",
       "17  []{a}<az:do:an>         0  az:do:an          a         \n",
       "17  []{a}<az:do:an>         0  az:do:an          a         \n",
       "\n",
       "                                               target correction_index  \\\n",
       "6   [No, confundas, a, Selena, Gomez, con, Selena,...          [2, 19]   \n",
       "9   [Un, Otro, famoso, actor, ,, que, llama, Nicol...         [11, 32]   \n",
       "17  [Princesa, Anna, y, sus, amigas, quieren, que,...          [8, 21]   \n",
       "17  [Pero, Eleanor, dice, su, secreto, a, su, amig...         [13, 30]   \n",
       "17  [Chiddi, enseña, a, Eleanor, sobre, cómo, ser,...          [2, 14]   \n",
       "\n",
       "   error_type                                             source  \n",
       "6       az:do  [No, confundas, [NONE], Selena, Gomez, con, Se...  \n",
       "9       az:do  [Un, Otro, famoso, actor, ,, que, llama, Nicol...  \n",
       "17      az:do  [Princesa, Anna, y, sus, amigas, quieren, que,...  \n",
       "17      az:do  [Pero, Eleanor, dice, su, secreto, a, su, amig...  \n",
       "17      az:do  [Chiddi, enseña, [NONE], Eleanor, sobre, cómo,...  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_combine[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HELWcxPdYlXe",
    "outputId": "95571f8f-98e2-4f4e-bcef-2e8087ff0ab2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['az:do' 'za:do' 'aa:do' 'aa:io' 'az:io' 'na:ps' 'ga:fm' 'ga:mf:det:aty'\n",
      " 'ga:mf' 'ig:f' 'na:sp' 'in:s' 'ga:na:fm:ps' 'ga:fm:det:aty' 'ga:ps'\n",
      " 'ig:e' 'ig:m' 'ga:na:mf:ps' 'ga:mf:noun:aty' 'ga:mf:adj:aty'\n",
      " 'ga:na:fm:sp' 'ga:na:mf:sp' 'in:p' 'ga:fm:adj:aty' 'ga:fm:noun:aty'\n",
      " 'ig:in:f:s' 'ga:fm:gat:noun' 'in:p:gat:noun' 'na:sp:gat:noun'\n",
      " 'ga:mf:pron:aty' 'in:s:gat:noun']\n",
      "error_type\n",
      "aa:do               12\n",
      "aa:io                1\n",
      "az:do              408\n",
      "az:io                7\n",
      "ga:fm             2018\n",
      "ga:fm:adj:aty        1\n",
      "ga:fm:det:aty       25\n",
      "ga:fm:gat:noun       1\n",
      "ga:fm:noun:aty       1\n",
      "ga:mf              650\n",
      "ga:mf:adj:aty       13\n",
      "ga:mf:det:aty      123\n",
      "ga:mf:noun:aty       1\n",
      "ga:mf:pron:aty       2\n",
      "ga:na:fm:ps        205\n",
      "ga:na:fm:sp         38\n",
      "ga:na:mf:ps         37\n",
      "ga:na:mf:sp         11\n",
      "ga:ps                1\n",
      "ig:e                22\n",
      "ig:f               102\n",
      "ig:in:f:s            2\n",
      "ig:m                34\n",
      "in:p                41\n",
      "in:p:gat:noun        3\n",
      "in:s               321\n",
      "in:s:gat:noun       29\n",
      "na:ps              855\n",
      "na:sp              208\n",
      "na:sp:gat:noun       2\n",
      "za:do               49\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df_combine.error_type.unique())\n",
    "print(df_combine.groupby(['error_type']).size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-pzNfGVHA_hm"
   },
   "outputs": [],
   "source": [
    "### Mapping the error type to three class\n",
    "\n",
    "error_mapping = {'az:do': 'article',\n",
    "'za:do': 'article',\n",
    "'aa:do': 'article',\n",
    "'aa:io': 'article',\n",
    "'az:io': 'article',\n",
    "\n",
    "'ga:fm': 'gender agreement',\n",
    "'ga:mf': 'gender agreement',\n",
    "'ga:mf:det:aty': 'gender agreement',\n",
    "'ga:mf:pron:aty': 'gender agreement',\n",
    "'ga:mf:adj:aty': 'gender agreement',\n",
    "'ga:mf:noun:aty': 'gender agreement',\n",
    "'ga:fm:det:aty': 'gender agreement',\n",
    "'ga:fm:adj:aty': 'gender agreement',\n",
    "'ga:fm:noun:aty': 'gender agreement',\n",
    "'ga:fm:gat:noun': 'gender agreement',\n",
    "'ga:ps': 'number agreement',  ## ps is related to plural and single\n",
    "\n",
    "'ga:na:mf:ps': 'gender and number agreement',\n",
    "'ga:na:fm:ps': 'gender and number agreement',\n",
    "'ga:na:fm:sp': 'gender and number agreement',\n",
    "'ga:na:mf:sp': 'gender and number agreement',\n",
    "\n",
    "'ig:f': 'gender agreement',\n",
    "'ig:e': 'gender agreement',\n",
    "'ig:m': 'gender agreement',\n",
    "'ig:in:f:s': 'gender agreement',\n",
    "\n",
    "'in:s': 'number agreement',\n",
    "'in:p': 'number agreement',\n",
    "'in:p:gat:noun': 'number agreement',\n",
    "'in:s:gat:noun':'number agreement',\n",
    "\n",
    "'na:sp': 'number agreement',\n",
    "'na:ps': 'number agreement',\n",
    "'na:sp:gat:noun': 'number agreement'}\n",
    "\n",
    "# Replace values in the 'ColumnToReplace' column using the mapping dictionary\n",
    "df_combine['new_error_type'] = df_combine['error_type'].replace(error_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e1CUUIDDC-Y5",
    "outputId": "05c6e6f3-08e7-480c-e445-b45d2e5d7bbb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['article' 'number agreement' 'gender agreement'\n",
      " 'gender and number agreement']\n",
      "new_error_type\n",
      "article                         477\n",
      "gender agreement               2995\n",
      "gender and number agreement     291\n",
      "number agreement               1460\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df_combine.new_error_type.unique())\n",
    "print(df_combine.groupby(['new_error_type']).size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ooo8vYD6bCHv"
   },
   "source": [
    "### split into 5 df for evident word so we can save the intermediate stages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "H-kyhNdibBRL"
   },
   "outputs": [],
   "source": [
    "df_combines = np.array_split(df_combine, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oy00D-YscWNr"
   },
   "outputs": [],
   "source": [
    "df_combine1, df_combine2, df_combine3,df_combine4, df_combine5 = df_combines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "U06_OMUkcb96",
    "outputId": "21c49576-50a2-4b47-b8e6-9c5331bb8305"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1045, 16)\n",
      "(1045, 16)\n",
      "(1045, 16)\n",
      "(1044, 16)\n",
      "(1044, 16)\n"
     ]
    }
   ],
   "source": [
    "print(df_combine1.shape)\n",
    "print(df_combine2.shape)\n",
    "print(df_combine3.shape)\n",
    "print(df_combine4.shape)\n",
    "print(df_combine5.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ajjLcT5fdKjB"
   },
   "outputs": [],
   "source": [
    "df_test1= df_combine1[:5]\n",
    "df_test2= df_combine2[:5]\n",
    "df_test3= df_combine3[:5]\n",
    "df_test4= df_combine4[:5]\n",
    "df_test5= df_combine5[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wVvzNUxZfJKb"
   },
   "source": [
    "## add evidence words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EaExLHrZfL_4",
    "outputId": "277442c1-2554-429b-9a05-3cfe676e9f8e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.3/43.3 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.0/77.0 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "llmx 0.0.15a0 requires cohere, which is not installed.\n",
      "llmx 0.0.15a0 requires tiktoken, which is not installed.\u001b[0m\u001b[31m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install langchain --quiet\n",
    "!pip install openai --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JNpONl8FfRRg"
   },
   "outputs": [],
   "source": [
    "os.environ['OPENAI_API_TOKEN'] = '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'\n",
    "openai_api_key = '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'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NUOW--_6fTWy",
    "outputId": "6d15fffa-f347-424d-8f93-199a7389345b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/langchain/llms/openai.py:216: UserWarning: You are trying to use a chat model. This way of initializing it is no longer supported. Instead, please use: `from langchain.chat_models import ChatOpenAI`\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/dist-packages/langchain/llms/openai.py:811: UserWarning: You are trying to use a chat model. This way of initializing it is no longer supported. Instead, please use: `from langchain.chat_models import ChatOpenAI`\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from langchain import LLMChain\n",
    "from langchain.llms import OpenAI\n",
    "\n",
    "gpt4 = OpenAI(model_name='gpt-4',\n",
    "                 openai_api_key=openai_api_key\n",
    "                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zdM7i54Fffhq"
   },
   "outputs": [],
   "source": [
    "# help function\n",
    "from google.colab import files\n",
    "\n",
    "def find_evidence_word(sentence, error, correction):\n",
    "  if error == '' or error == ' ':\n",
    "    error = '[NONE]'\n",
    "    # print(f'give an index number starting from 0 for each word in this sentence, \"{sentence}\" This sentence has a grammatical error, where \"{error}\" should be \"{correction}\". What are the evidence words for this grammatical error?  Please respond in the \"evidence words =\", \"indexes = \" format in the order of the sentence.')\n",
    "  return gpt4(f'give an index number starting from 0 for each word in this sentence, \"{sentence}\" This sentence has a grammatical error, where \"{error}\" should be \"{correction}\". What are the evidence words for this grammatical error?  Please respond in the \"evidence words =\", \"indexes = \" format in the order of the sentence.')\n",
    "\n",
    "# put the source in list into a sentence\n",
    "def joiner(lists):\n",
    "  return ' '.join(lists)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BLsnqwMSdcFA"
   },
   "source": [
    "#### example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "Tm74OSP1lJqW",
    "outputId": "aa9a205a-1d0b-422d-91c1-8dd24be4c959"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'evidence words = No, confundas, Selena, Gomez, con, Selena, Quintanilla\\nindexes = 0, 1, 2, 3, 4, 5, 6'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_evidence_word('No confundas [NONE] Selena Gomez con Selena Quintanilla , de quien Selena Gomez es nombrada .', '', 'a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "snRKIiH8kxyR",
    "outputId": "ebc9d55c-78bf-49f3-84b9-4cc9c8a1fa9b"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'evidence words = Selena Gomez, es, nombrada\\nindexes = 2, 13, 14'"
      ]
     },
     "execution_count": 420,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_evidence_word('No confundas Selena Gomez con Selena Quintanilla , de quien Selena Gomez es nombrada .', ' ', 'a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "33vSbM2blgX2",
    "outputId": "bb40caa1-e505-4d75-a6d3-6b136377d0fb"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'evidence words = [\"confundas\", \"[NONE]\", \"Selena\", \"Gomez\", \"con\", \"Selena\", \"Quintanilla\"]\\nindexes = [1, 2, 3, 4, 5, 6, 7]'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_evidence_word('No confundas [NONE] Selena Gomez con Selena Quintanilla , de quien Selena Gomez es nombrada .', '[NONE]', 'a')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6pM5vE2edfti"
   },
   "source": [
    "### add evidence words in batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 228
    },
    "id": "QXbX-Vw-daRU",
    "outputId": "e45fbe87-4538-4e1e-955b-e5772d2c7e11"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_cc9b482c-3a97-4742-829f-f3062b99328f\", \"combine_df_1.csv\", 764533)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_6e00efbd-fe1b-4b67-85a9-51499928f6be\", \"combine_df_2.csv\", 756098)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 05:53:06 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81d91956da45b130-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised Timeout: Request timed out: HTTPSConnectionPool(host='api.openai.com', port=443): Read timed out. (read timeout=600).\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 06:15:13 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81d939bf5ab2b160-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 06:25:43 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81d9491ebc60458a-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 06:36:45 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81d9594bea752450-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised Timeout: Request timed out: HTTPSConnectionPool(host='api.openai.com', port=443): Read timed out. (read timeout=600).\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 06:58:53 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81d979b37acdb0c4-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_581adeac-a8a6-4d1d-9e61-511a7336841f\", \"combine_df_3.csv\", 784321)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 07:23:33 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81d99dd7add9b066-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 07:34:49 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81d9ae597825b106-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_ac366a31-b3f0-4525-aa8b-c9b4d84f8230\", \"combine_df_4.csv\", 886978)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 08:30:40 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81da0027fae5ad9a-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n",
      "WARNING:langchain.llms.base:Retrying langchain.llms.openai.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Sun, 29 Oct 2023 08:42:19 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '81da11371da0b124-ATL', 'alt-svc': 'h3=\":443\"; ma=86400'}.\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_f9c5925f-8fc5-4888-81da-e11fe173990f\", \"combine_df_5.csv\", 874697)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "counter = 1\n",
    "for i in [df_combine1,df_combine2,df_combine3,df_combine4,df_combine5]:\n",
    "  i['source_sentence'] = i['source'].apply(joiner)\n",
    "  i['raw_evidence'] = i.apply(lambda row : find_evidence_word(row['source_sentence'], row['error'], row['correction']), axis=1)\n",
    "  # saving to csv\n",
    "  i.to_csv(f'combine_df_{counter}.csv')\n",
    "  # download\n",
    "  files.download(f'combine_df_{counter}.csv')\n",
    "  counter += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8QHhbHDefSUi"
   },
   "source": [
    "### processing the data from gpt model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "sabl2JGOwim-"
   },
   "outputs": [],
   "source": [
    "pd.options.display.max_colwidth = 400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "id": "ZRNiIeKXfhPe"
   },
   "outputs": [],
   "source": [
    "df = pd.concat([pd.read_csv('combine_df_1.csv'),pd.read_csv('combine_df_2.csv'),pd.read_csv('combine_df_3.csv'),pd.read_csv('combine_df_4.csv'),pd.read_csv('combine_df_5.csv')], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BcWo2GaZzIrt",
    "outputId": "14b5bdb4-d689-482f-b415-cce42c5c538d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'paragraph_id', 'sentence_id', 'course', 'essay_new',\n",
       "       'annotated_sentence', 'anno_error_cnt', 'full_error_tag', 'order_tag',\n",
       "       'error_tag', 'correction', 'error', 'target', 'correction_index',\n",
       "       'error_type', 'source', 'new_error_type', 'source_sentence',\n",
       "       'raw_evidence'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3hoo2xGs9CKG",
    "outputId": "8ea0efb1-f725-4f81-8ef7-cb03fd9031f3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5223, 19)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "Se_kl1XSzN-m",
    "outputId": "db339fab-d993-40ba-8e64-f26e0dca39ca"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-93cafd87-fff5-4af7-92f4-cb87730fab56\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>anno_error_cnt</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>order_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>error_type</th>\n",
       "      <th>source</th>\n",
       "      <th>new_error_type</th>\n",
       "      <th>source_sentence</th>\n",
       "      <th>raw_evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['No', 'confundas', 'a', 'Selena', 'Gomez', 'con', 'Selena', 'Quintanilla', ',', 'de', 'quien', 'Selena', 'Gomez', 'es', 'nombrada', '.']</td>\n",
       "      <td>[2, 19]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>['No', 'confundas', '[NONE]', 'Selena', 'Gomez', 'con', 'Selena', 'Quintanilla', ',', 'de', 'quien', 'Selena', 'Gomez', 'es', 'nombrada', '.']</td>\n",
       "      <td>article</td>\n",
       "      <td>No confundas [NONE] Selena Gomez con Selena Quintanilla , de quien Selena Gomez es nombrada .</td>\n",
       "      <td>evidence words = \"confundas\", \"[NONE]\", \"Selena Gomez\", \"con\", \"Selena Quintanilla\"\\nindexes = 1, 2, 3, 4, 5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}&lt;az:do:an&gt; Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Un', 'Otro', 'famoso', 'actor', ',', 'que', 'llama', 'Nicolas', 'Cage', ',', 'ayudo', 'a', 'Johnny', 'Depp', 'empezar', 'su', 'profesión', 'de', 'actor', '.']</td>\n",
       "      <td>[11, 32]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>['Un', 'Otro', 'famoso', 'actor', ',', 'que', 'llama', 'Nicolas', 'Cage', ',', 'ayudo', '[NONE]', 'Johnny', 'Depp', 'empezar', 'su', 'profesión', 'de', 'actor', '.']</td>\n",
       "      <td>article</td>\n",
       "      <td>Un Otro famoso actor , que llama Nicolas Cage , ayudo [NONE] Johnny Depp empezar su profesión de actor .</td>\n",
       "      <td>evidence words = Nicolas Cage, ayudo, Johnny Depp, empezar\\nindexes = 7, 9, 11, 12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>0e02104bc91f910260846ddba52137596f5b83525260c1e2bea381b74804d410</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden Princesa Elsa.</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden []{a}&lt;az:do:an&gt; Princesa Elsa.</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Princesa', 'Anna', 'y', 'sus', 'amigas', 'quieren', 'que', 'ayuden', 'a', 'Princesa', 'Elsa', '.']</td>\n",
       "      <td>[8, 21]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>['Princesa', 'Anna', 'y', 'sus', 'amigas', 'quieren', 'que', 'ayuden', '[NONE]', 'Princesa', 'Elsa', '.']</td>\n",
       "      <td>article</td>\n",
       "      <td>Princesa Anna y sus amigas quieren que ayuden [NONE] Princesa Elsa .</td>\n",
       "      <td>evidence words = Princesa, que, ayuden, Princesa\\nindexes = 0, 5, 6, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>dda3163fe647d016c4f212fd794343bd1b2446243a6ac3f7ebfb6ad30ae6c65a</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Pero Eleanor dice su secreto a su amigo Chiddi y Chiddi quiere ayudar Eleanor.</td>\n",
       "      <td>Pero Eleanor dice su secreto a su amigo Chiddi y Chiddi quiere ayudar []{a}&lt;az:do:an&gt; Eleanor.</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Pero', 'Eleanor', 'dice', 'su', 'secreto', 'a', 'su', 'amigo', 'Chiddi', 'y', 'Chiddi', 'quiere', 'ayudar', 'a', 'Eleanor', '.']</td>\n",
       "      <td>[13, 30]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>['Pero', 'Eleanor', 'dice', 'su', 'secreto', 'a', 'su', 'amigo', 'Chiddi', 'y', 'Chiddi', 'quiere', 'ayudar', '[NONE]', 'Eleanor', '.']</td>\n",
       "      <td>article</td>\n",
       "      <td>Pero Eleanor dice su secreto a su amigo Chiddi y Chiddi quiere ayudar [NONE] Eleanor .</td>\n",
       "      <td>evidence words = Pero, ayudar, Eleanor\\nindexes = 0, 12, 14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>abb0ff0f493ec9cabdf7eaeb7b0e0078879fbe59f5768fc81130028ceb015690</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Chiddi enseña Eleanor sobre cómo ser una persona buena.</td>\n",
       "      <td>Chiddi enseña []{a}&lt;az:do:an&gt; Eleanor sobre cómo ser una persona buena.</td>\n",
       "      <td>1</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Chiddi', 'enseña', 'a', 'Eleanor', 'sobre', 'cómo', 'ser', 'una', 'persona', 'buena', '.']</td>\n",
       "      <td>[2, 14]</td>\n",
       "      <td>az:do</td>\n",
       "      <td>['Chiddi', 'enseña', '[NONE]', 'Eleanor', 'sobre', 'cómo', 'ser', 'una', 'persona', 'buena', '.']</td>\n",
       "      <td>article</td>\n",
       "      <td>Chiddi enseña [NONE] Eleanor sobre cómo ser una persona buena .</td>\n",
       "      <td>evidence words = Chiddi, enseña, [NONE], Eleanor, sobre, cómo, ser, una, persona, buena\\nindexes = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-93cafd87-fff5-4af7-92f4-cb87730fab56')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-93cafd87-fff5-4af7-92f4-cb87730fab56 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-93cafd87-fff5-4af7-92f4-cb87730fab56');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-b0bf388e-ce6c-4fc9-b7ec-b4152876bd80\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b0bf388e-ce6c-4fc9-b7ec-b4152876bd80')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-b0bf388e-ce6c-4fc9-b7ec-b4152876bd80 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "   Unnamed: 0  paragraph_id  \\\n",
       "0           6            13   \n",
       "1           9            18   \n",
       "2          17            28   \n",
       "3          17            28   \n",
       "4          17            28   \n",
       "\n",
       "                                                        sentence_id course  \\\n",
       "0  452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e  SPA 1   \n",
       "1  e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b  SPA 2   \n",
       "2  0e02104bc91f910260846ddba52137596f5b83525260c1e2bea381b74804d410  SPA 3   \n",
       "3  dda3163fe647d016c4f212fd794343bd1b2446243a6ac3f7ebfb6ad30ae6c65a  SPA 3   \n",
       "4  abb0ff0f493ec9cabdf7eaeb7b0e0078879fbe59f5768fc81130028ceb015690  SPA 3   \n",
       "\n",
       "                                                                                        essay_new  \\\n",
       "0            No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1  Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.   \n",
       "2                                    Princesa Anna y sus amigas quieren que ayuden Princesa Elsa.   \n",
       "3                  Pero Eleanor dice su secreto a su amigo Chiddi y Chiddi quiere ayudar Eleanor.   \n",
       "4                                         Chiddi enseña Eleanor sobre cómo ser una persona buena.   \n",
       "\n",
       "                                                                                               annotated_sentence  \\\n",
       "0            No confundas []{a}<az:do:an> Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1  Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}<az:do:an> Johnny Depp empezar su profesión de actor.   \n",
       "2                                    Princesa Anna y sus amigas quieren que ayuden []{a}<az:do:an> Princesa Elsa.   \n",
       "3                  Pero Eleanor dice su secreto a su amigo Chiddi y Chiddi quiere ayudar []{a}<az:do:an> Eleanor.   \n",
       "4                                         Chiddi enseña []{a}<az:do:an> Eleanor sobre cómo ser una persona buena.   \n",
       "\n",
       "   anno_error_cnt   full_error_tag  order_tag error_tag correction error  \\\n",
       "0               1  []{a}<az:do:an>          0  az:do:an          a   NaN   \n",
       "1               1  []{a}<az:do:an>          0  az:do:an          a   NaN   \n",
       "2               1  []{a}<az:do:an>          0  az:do:an          a   NaN   \n",
       "3               1  []{a}<az:do:an>          0  az:do:an          a   NaN   \n",
       "4               1  []{a}<az:do:an>          0  az:do:an          a   NaN   \n",
       "\n",
       "                                                                                                                                                             target  \\\n",
       "0                         ['No', 'confundas', 'a', 'Selena', 'Gomez', 'con', 'Selena', 'Quintanilla', ',', 'de', 'quien', 'Selena', 'Gomez', 'es', 'nombrada', '.']   \n",
       "1  ['Un', 'Otro', 'famoso', 'actor', ',', 'que', 'llama', 'Nicolas', 'Cage', ',', 'ayudo', 'a', 'Johnny', 'Depp', 'empezar', 'su', 'profesión', 'de', 'actor', '.']   \n",
       "2                                                              ['Princesa', 'Anna', 'y', 'sus', 'amigas', 'quieren', 'que', 'ayuden', 'a', 'Princesa', 'Elsa', '.']   \n",
       "3                                ['Pero', 'Eleanor', 'dice', 'su', 'secreto', 'a', 'su', 'amigo', 'Chiddi', 'y', 'Chiddi', 'quiere', 'ayudar', 'a', 'Eleanor', '.']   \n",
       "4                                                                      ['Chiddi', 'enseña', 'a', 'Eleanor', 'sobre', 'cómo', 'ser', 'una', 'persona', 'buena', '.']   \n",
       "\n",
       "  correction_index error_type  \\\n",
       "0          [2, 19]      az:do   \n",
       "1         [11, 32]      az:do   \n",
       "2          [8, 21]      az:do   \n",
       "3         [13, 30]      az:do   \n",
       "4          [2, 14]      az:do   \n",
       "\n",
       "                                                                                                                                                                  source  \\\n",
       "0                         ['No', 'confundas', '[NONE]', 'Selena', 'Gomez', 'con', 'Selena', 'Quintanilla', ',', 'de', 'quien', 'Selena', 'Gomez', 'es', 'nombrada', '.']   \n",
       "1  ['Un', 'Otro', 'famoso', 'actor', ',', 'que', 'llama', 'Nicolas', 'Cage', ',', 'ayudo', '[NONE]', 'Johnny', 'Depp', 'empezar', 'su', 'profesión', 'de', 'actor', '.']   \n",
       "2                                                              ['Princesa', 'Anna', 'y', 'sus', 'amigas', 'quieren', 'que', 'ayuden', '[NONE]', 'Princesa', 'Elsa', '.']   \n",
       "3                                ['Pero', 'Eleanor', 'dice', 'su', 'secreto', 'a', 'su', 'amigo', 'Chiddi', 'y', 'Chiddi', 'quiere', 'ayudar', '[NONE]', 'Eleanor', '.']   \n",
       "4                                                                      ['Chiddi', 'enseña', '[NONE]', 'Eleanor', 'sobre', 'cómo', 'ser', 'una', 'persona', 'buena', '.']   \n",
       "\n",
       "  new_error_type  \\\n",
       "0        article   \n",
       "1        article   \n",
       "2        article   \n",
       "3        article   \n",
       "4        article   \n",
       "\n",
       "                                                                                            source_sentence  \\\n",
       "0             No confundas [NONE] Selena Gomez con Selena Quintanilla , de quien Selena Gomez es nombrada .   \n",
       "1  Un Otro famoso actor , que llama Nicolas Cage , ayudo [NONE] Johnny Depp empezar su profesión de actor .   \n",
       "2                                      Princesa Anna y sus amigas quieren que ayuden [NONE] Princesa Elsa .   \n",
       "3                    Pero Eleanor dice su secreto a su amigo Chiddi y Chiddi quiere ayudar [NONE] Eleanor .   \n",
       "4                                           Chiddi enseña [NONE] Eleanor sobre cómo ser una persona buena .   \n",
       "\n",
       "                                                                                                                      raw_evidence  \n",
       "0                     evidence words = \"confundas\", \"[NONE]\", \"Selena Gomez\", \"con\", \"Selena Quintanilla\"\\nindexes = 1, 2, 3, 4, 5  \n",
       "1                                               evidence words = Nicolas Cage, ayudo, Johnny Depp, empezar\\nindexes = 7, 9, 11, 12  \n",
       "2                                                           evidence words = Princesa, que, ayuden, Princesa\\nindexes = 0, 5, 6, 8  \n",
       "3                                                                      evidence words = Pero, ayudar, Eleanor\\nindexes = 0, 12, 14  \n",
       "4  evidence words = Chiddi, enseña, [NONE], Eleanor, sobre, cómo, ser, una, persona, buena\\nindexes = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "id": "qkaAXztYyr85"
   },
   "outputs": [],
   "source": [
    "df_final = df[['paragraph_id','sentence_id', 'course','essay_new', 'annotated_sentence', 'full_error_tag','error_tag','correction','error','target','source', 'correction_index', 'new_error_type','raw_evidence']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0FCpg-bFyWoB",
    "outputId": "aca23528-f3bc-4d41-bff6-d02e90a0f4b2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-65-43b466fbd449>:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final['target'] = df_final['target'].apply(ast.literal_eval)\n",
      "<ipython-input-65-43b466fbd449>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final['source'] = df_final['source'].apply(ast.literal_eval)\n",
      "<ipython-input-65-43b466fbd449>:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_final['correction_index'] = df_final['correction_index'].apply(ast.literal_eval)\n"
     ]
    }
   ],
   "source": [
    "import ast\n",
    "#turn string back to list\n",
    "df_final['target'] = df_final['target'].apply(ast.literal_eval)\n",
    "df_final['source'] = df_final['source'].apply(ast.literal_eval)\n",
    "df_final['correction_index'] = df_final['correction_index'].apply(ast.literal_eval)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "Phl4OJX0qgai"
   },
   "outputs": [],
   "source": [
    "def get_evidence_words_list(sentence):\n",
    "  sentence= sentence.replace('\\n', ' ')\n",
    "  sentence= sentence.replace('[', '')\n",
    "  sentence= sentence.replace(']', '')\n",
    "  sentence= sentence.replace('\"', '')\n",
    "  sentence= sentence.replace('\\.', '')\n",
    "  words = re.match(r'evidence words = (.+?) indexes = (.+)', sentence).group(1)\n",
    "  words= words.replace(',', '')\n",
    "  return [i.strip() for i in words.split(' ')]\n",
    "\n",
    "\n",
    "def get_evidence_index_list(sentence):\n",
    "  sentence= sentence.replace('\\n', ' ')\n",
    "  sentence= sentence.replace('[', '')\n",
    "  sentence= sentence.replace(']', '')\n",
    "  sentence= sentence.replace('\"', '')\n",
    "  sentence= sentence.replace('.', '')\n",
    "  sentence= sentence.replace('-', ',')\n",
    "  words = re.match(r'evidence words = (.+?) indexes = (.+)', sentence).group(2)\n",
    "  return [int(i.strip()) for i in words.split(',') if i.strip().isdigit() ]\n",
    "\n",
    "def check_if_same_len(list1, list2):\n",
    "  if len(list1)== len(list2):\n",
    "    return True\n",
    "  else:\n",
    "    return False\n",
    "\n",
    "def remove_correction_index(correction_index, evidence_index):\n",
    "  if correction_index[0] in evidence_index:\n",
    "    return evidence_index.remove(correction_index[0])\n",
    "  return evidence_index\n",
    "\n",
    "def final_evidence_index(evidence_index, source):\n",
    "  return evidence_index + [i+len(source)+1 for i in evidence_index]\n",
    "\n",
    "#when the index and word is not the same length\n",
    "# dealing with the not order one/ but cannot have two of the same words\n",
    "# def find_evidence_index_from_list(evidence_words, source, correction_index):\n",
    "#   indexes = {}\n",
    "#   index_array = []\n",
    "#   correction = correction_index[0]\n",
    "#   for i in range(len(evidence_words)):\n",
    "#     for x in range(len(source)):\n",
    "#       if evidence_words[i] == source[x]:\n",
    "#         if indexes.get(evidence_words[i]):\n",
    "#           indexes.get(evidence_words[i]).append(x)\n",
    "#         else:\n",
    "#           indexes[evidence_words[i]] = [x]\n",
    "\n",
    "#   for i in indexes.keys():\n",
    "#     if len(indexes[i]) == 1:\n",
    "#       index_array.append(indexes[i][0])\n",
    "#     else:\n",
    "#       closest_array = 99\n",
    "#       diff = 99\n",
    "#       for x in indexes[i]:\n",
    "#         if abs(x - correction) < diff:\n",
    "#           close_array = x\n",
    "#       index_array.append(x)\n",
    "#   return sorted(index_array)\n",
    "\n",
    "## assume gpt will return in order\n",
    "def find_evidence_index_from_list(evidence_words, source, correction_index):\n",
    "  indexes = []\n",
    "  mid = int(correction_index[0])\n",
    "  pointer2 = 0\n",
    "  lower = mid - len(evidence_words)\n",
    "  higher = mid + len(evidence_words)\n",
    "  if lower < 0:\n",
    "    lower = 0\n",
    "  if higher > len(source)-1:\n",
    "    higher = len(source)-1\n",
    "  source_pointer = lower\n",
    "  while source_pointer < higher and pointer2 < len(evidence_words):\n",
    "\n",
    "    if evidence_words[pointer2] ==source[source_pointer]:\n",
    "      indexes.append(source_pointer)\n",
    "      source_pointer +=1\n",
    "      pointer2 += 1\n",
    "    elif evidence_words[pointer2] == 'NONE':\n",
    "      pointer2 += 1\n",
    "    else:\n",
    "      source_pointer +=1\n",
    "  return indexes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "id": "-nFJTc5x6vdc"
   },
   "outputs": [],
   "source": [
    "def is_correct_format(sentence):\n",
    "  sentence= sentence.replace('\\n', ' ')\n",
    "  sentence= sentence.replace('[', '')\n",
    "  sentence= sentence.replace(']', '')\n",
    "  sentence= sentence.replace('\"', '')\n",
    "  if re.match(r'evidence words = (.+?) indexes = (.+)', sentence):\n",
    "    return True\n",
    "  else:\n",
    "    return False\n",
    "well_format = df_final['raw_evidence'].apply(is_correct_format)\n",
    "\n",
    "# Filter rows and move them to the other dataset\n",
    "df_final_gpt_well_format = df_final[well_format]\n",
    "df_final_gpt_bad_format = df_final[~well_format]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "id": "iG4t5YLP8kXT"
   },
   "outputs": [],
   "source": [
    "# df_final_gpt_bad_format.to_csv('bad_format.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "id": "dXhXubTPlNSo"
   },
   "outputs": [],
   "source": [
    "corrected_format = pd.read_csv('df_final_gpt_bad_format_corrected.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "id": "lb90c3NGnjTA"
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "#turn string back to list\n",
    "corrected_format['target'] = corrected_format['target'].apply(ast.literal_eval)\n",
    "corrected_format['source'] = corrected_format['source'].apply(ast.literal_eval)\n",
    "corrected_format['correction_index'] = corrected_format['correction_index'].apply(ast.literal_eval)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "T7NjE5btlXsN",
    "outputId": "202b0be4-abdd-4c47-e9cc-99c66315f43f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 15)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "well_format = corrected_format['raw_evidence'].apply(is_correct_format)\n",
    "corrected_format[~well_format].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "id": "EQMKpfdH9olw"
   },
   "outputs": [],
   "source": [
    "#combine well and bad format\n",
    "df_final =  pd.concat([df_final_gpt_well_format,corrected_format])  ## to do"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "id": "gD7x3OYIt7Nz"
   },
   "outputs": [],
   "source": [
    "df_final['evident_words'] = df_final['raw_evidence'].apply(get_evidence_words_list)\n",
    "df_final['evident_index'] = df_final['raw_evidence'].apply(get_evidence_index_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "id": "NU1x5oaaok6S"
   },
   "outputs": [],
   "source": [
    "#check if the index and the words are the same length\n",
    "is_same = df_final.apply(lambda row : check_if_same_len(row['evident_words'], row['evident_index']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "id": "JccOnjSKpESR"
   },
   "outputs": [],
   "source": [
    "# get the correct formatted evident words\n",
    "df_correct = df_final[is_same]\n",
    "# get the incorrect formatted evident words\n",
    "df_incorrect = df_final[~is_same]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kG22Y8sp__r5",
    "outputId": "fcd1dd6d-1621-400a-c8dd-e94f522c8225"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(246, 17)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_incorrect.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "Vm9SfnBebSuJ",
    "outputId": "77313711-d019-49f7-d06d-91e4783db8c8"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-1d7bae66-09ae-40e7-b5cf-70ef3dc7eeff\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>source</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>new_error_type</th>\n",
       "      <th>raw_evidence</th>\n",
       "      <th>evident_words</th>\n",
       "      <th>evident_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[2, 19]</td>\n",
       "      <td>article</td>\n",
       "      <td>evidence words = \"confundas\", \"[NONE]\", \"Selena Gomez\", \"con\", \"Selena Quintanilla\"\\nindexes = 1, 2, 3, 4, 5</td>\n",
       "      <td>[confundas, NONE, Selena, Gomez, con, Selena, Quintanilla]</td>\n",
       "      <td>[1, 2, 3, 4, 5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}&lt;az:do:an&gt; Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[11, 32]</td>\n",
       "      <td>article</td>\n",
       "      <td>evidence words = Nicolas Cage, ayudo, Johnny Depp, empezar\\nindexes = 7, 9, 11, 12</td>\n",
       "      <td>[Nicolas, Cage, ayudo, Johnny, Depp, empezar]</td>\n",
       "      <td>[7, 9, 11, 12]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>71</td>\n",
       "      <td>32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Yo miro él en la television.</td>\n",
       "      <td>Yo miro []{a}&lt;az:do:an&gt; él en la television.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, miro, a, él, en, la, television, .]</td>\n",
       "      <td>[Yo, miro, [NONE], él, en, la, television, .]</td>\n",
       "      <td>[2, 11]</td>\n",
       "      <td>article</td>\n",
       "      <td>evidence words = Yo, miro, él, en, la, television \\nindexes = 0, 1, 3, 4, 5, 6</td>\n",
       "      <td>[Yo, miro, él, en, la, television, ]</td>\n",
       "      <td>[0, 1, 3, 4, 5, 6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>93</td>\n",
       "      <td>b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo quiero conocer Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>Yo quiero conocer []{a}&lt;az:do:an&gt; Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[3, 16]</td>\n",
       "      <td>article</td>\n",
       "      <td>evidence words = \"conocer\", \"Taylor Swift\"\\nindexes = 2, 4</td>\n",
       "      <td>[conocer, Taylor, Swift]</td>\n",
       "      <td>[2, 4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>139</td>\n",
       "      <td>7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>Yo admiro []{a}&lt;az:do:an&gt; Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[2, 18]</td>\n",
       "      <td>article</td>\n",
       "      <td>evidence words = admiro, Ian Somerhalder\\nindexes = 1, 3</td>\n",
       "      <td>[admiro, Ian, Somerhalder]</td>\n",
       "      <td>[1, 3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875</th>\n",
       "      <td>1712</td>\n",
       "      <td>f36b65866ccb9a75ea6094de6703de5078ac67b8b90aa645f167c5d3a5cf7268</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Tu nadas en el mar, pescadas, comes mucho alimentos diferentes, y caminas tu perro en la playa.</td>\n",
       "      <td>Tu nadas en el mar, [pescadas]{pescados}&lt;ig:f:noun:an&gt;, comes [mucho]{muchos}&lt;na:ps:det:inan&gt; alimentos diferentes, y caminas tu perro en la playa.</td>\n",
       "      <td>[pescadas]{pescados}&lt;ig:f:noun:an&gt;</td>\n",
       "      <td>ig:f:noun:an</td>\n",
       "      <td>pescados</td>\n",
       "      <td>pescadas</td>\n",
       "      <td>[Tu, nadas, en, el, mar, ,, pescados, ,, comes, muchos, alimentos, diferentes, ,, y, caminas, tu, perro, en, la, playa, .]</td>\n",
       "      <td>[Tu, nadas, en, el, mar, ,, pescadas, ,, comes, muchos, alimentos, diferentes, ,, y, caminas, tu, perro, en, la, playa, .]</td>\n",
       "      <td>[6, 28]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>evidence words = \"mar , pescadas ,\", \"indexes = 4, 5, 6\"</td>\n",
       "      <td>[mar, , pescadas, ]</td>\n",
       "      <td>[4, 5, 6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>918</th>\n",
       "      <td>1724</td>\n",
       "      <td>4ba303022994dcd9e603f7ff90524275eea9d09b29b672188ce5c57b5e63ce0d</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Entonces, Hawaii es la casa a muchos frutas tropicales, plants unicos, y animales que esta en la isla solamente.</td>\n",
       "      <td>Entonces, Hawaii es la casa a [muchos]{muchas}&lt;ga:fm:det:inan&gt; frutas tropicales, plants [únicos]{únicas}&lt;ga:fm:adj:an&gt;, y animales que esta en la isla solamente.</td>\n",
       "      <td>[únicos]{únicas}&lt;ga:fm:adj:an&gt;</td>\n",
       "      <td>ga:fm:adj:an</td>\n",
       "      <td>únicas</td>\n",
       "      <td>únicos</td>\n",
       "      <td>[Entonces, ,, Hawaii, es, la, casa, a, muchas, frutas, tropicales, ,, plants, únicas, ,, y, animales, que, esta, en, la, isla, solamente, .]</td>\n",
       "      <td>[Entonces, ,, Hawaii, es, la, casa, a, muchas, frutas, tropicales, ,, plants, únicos, ,, y, animales, que, esta, en, la, isla, solamente, .]</td>\n",
       "      <td>[12, 36]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>evidence words = \", plants únicos ,\", \"indexes = 13, 14, 15\"</td>\n",
       "      <td>[, plants, únicos, ]</td>\n",
       "      <td>[13, 14, 15]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>341</td>\n",
       "      <td>47fdfb3f199222f4e62dcd01d3d7e7682bdf16c76713f04aa320d7dd311259cf</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Es importante que las gentes de multiples culturas del mundo pueden sentirses representados.</td>\n",
       "      <td>Es importante que las [gentes]{gente}&lt;na:sp:noun:an&gt; de multiples culturas del mundo pueden sentirses [representados]{representada}&lt;ga:fm:adj:an&gt;.</td>\n",
       "      <td>[representados]{representada}&lt;ga:fm:adj:an&gt;</td>\n",
       "      <td>ga:fm:adj:an</td>\n",
       "      <td>representada</td>\n",
       "      <td>representados</td>\n",
       "      <td>[Es, importante, que, las, gente, de, multiples, culturas, del, mundo, pueden, sentirses, representada, .]</td>\n",
       "      <td>[Es, importante, que, las, gente, de, multiples, culturas, del, mundo, pueden, sentirses, representados, .]</td>\n",
       "      <td>[12, 27]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>evidence words = \"la gente\", \"representados\"\\nindexes = 3, 11</td>\n",
       "      <td>[la, gente, representados]</td>\n",
       "      <td>[3, 11]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>1129</td>\n",
       "      <td>67b600f36f1d3f5c11314e1e7a765364ad64d20bb909d8e16684e383fab607f2</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>El cuarto sería una suite con un baño muy grande, una cama cómoda, y una sala con un televisión muy nuevo.</td>\n",
       "      <td>El cuarto sería una suite con un baño muy grande, una cama cómoda, y una sala con [un]{una}&lt;ga:fm:det:inan&gt; televisión muy [nuevo]{nueva}&lt;ga:fm:adj:inan&gt;.</td>\n",
       "      <td>[nuevo]{nueva}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>ga:fm:adj:inan</td>\n",
       "      <td>nueva</td>\n",
       "      <td>nuevo</td>\n",
       "      <td>[El, cuarto, sería, una, suite, con, un, baño, muy, grande, ,, una, cama, cómoda, ,, y, una, sala, con, una, televisión, muy, nueva, .]</td>\n",
       "      <td>[El, cuarto, sería, una, suite, con, un, baño, muy, grande, ,, una, cama, cómoda, ,, y, una, sala, con, una, televisión, muy, nuevo, .]</td>\n",
       "      <td>[22, 47]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>evidence words = televisión, muy, nuevo \\nindexes = 23, 24, 25</td>\n",
       "      <td>[televisión, muy, nuevo, ]</td>\n",
       "      <td>[23, 24, 25]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1028</th>\n",
       "      <td>1360</td>\n",
       "      <td>8a4959c8f8976fa8661677f5bf0113a1dfb282852376f660ac858c4e27d31f4f</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Durante mi vacación perfecto no hay nada para hacer o nada para mi a preocupada a, especialmente tarea o estudiado, solamente necesito relajarme.</td>\n",
       "      <td>Durante mi [vacación]{vacaciones}&lt;in:s:noun:inan&gt; [perfecto]{perfecta}&lt;ga:fm:adj:inan&gt; no hay nada para hacer o nada para mi a preocupada a, especialmente tarea o estudiado, solamente necesito relajarme.</td>\n",
       "      <td>[vacación]{vacaciones}&lt;in:s:noun:inan&gt;</td>\n",
       "      <td>in:s:noun:inan</td>\n",
       "      <td>vacaciones</td>\n",
       "      <td>vacación</td>\n",
       "      <td>[Durante, mi, vacaciones, perfecta, no, hay, nada, para, hacer, o, nada, para, mi, a, preocupada, a, ,, especialmente, tarea, o, estudiado, ,, solamente, necesito, relajarme, .]</td>\n",
       "      <td>[Durante, mi, vacación, perfecta, no, hay, nada, para, hacer, o, nada, para, mi, a, preocupada, a, ,, especialmente, tarea, o, estudiado, ,, solamente, necesito, relajarme, .]</td>\n",
       "      <td>[2, 29]</td>\n",
       "      <td>number agreement</td>\n",
       "      <td>evidence words = \"mi\", \"vacación\" \\nindexes = 1, 2</td>\n",
       "      <td>[mi, vacación, ]</td>\n",
       "      <td>[1, 2]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>241 rows × 16 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1d7bae66-09ae-40e7-b5cf-70ef3dc7eeff')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-1d7bae66-09ae-40e7-b5cf-70ef3dc7eeff button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-1d7bae66-09ae-40e7-b5cf-70ef3dc7eeff');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-ee0d3272-9030-4cb4-a68d-6653bfcc4036\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ee0d3272-9030-4cb4-a68d-6653bfcc4036')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-ee0d3272-9030-4cb4-a68d-6653bfcc4036 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "      paragraph_id  \\\n",
       "0               13   \n",
       "1               18   \n",
       "10              71   \n",
       "13              93   \n",
       "32             139   \n",
       "...            ...   \n",
       "875           1712   \n",
       "918           1724   \n",
       "940            341   \n",
       "992           1129   \n",
       "1028          1360   \n",
       "\n",
       "                                                           sentence_id course  \\\n",
       "0     452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e  SPA 1   \n",
       "1     e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b  SPA 2   \n",
       "10    32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6  SPA 1   \n",
       "13    b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2  SPA 2   \n",
       "32    7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f  SPA 2   \n",
       "...                                                                ...    ...   \n",
       "875   f36b65866ccb9a75ea6094de6703de5078ac67b8b90aa645f167c5d3a5cf7268  SPA 3   \n",
       "918   4ba303022994dcd9e603f7ff90524275eea9d09b29b672188ce5c57b5e63ce0d  SPA 3   \n",
       "940   47fdfb3f199222f4e62dcd01d3d7e7682bdf16c76713f04aa320d7dd311259cf  SPA 3   \n",
       "992   67b600f36f1d3f5c11314e1e7a765364ad64d20bb909d8e16684e383fab607f2  SPA 3   \n",
       "1028  8a4959c8f8976fa8661677f5bf0113a1dfb282852376f660ac858c4e27d31f4f  SPA 1   \n",
       "\n",
       "                                                                                                                                              essay_new  \\\n",
       "0                                                                  No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1                                                        Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                                                                         Yo miro él en la television.   \n",
       "13                                                                                           Yo quiero conocer Taylor Swift porque ella no es aburrida.   \n",
       "32                                                                  Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "...                                                                                                                                                 ...   \n",
       "875                                                     Tu nadas en el mar, pescadas, comes mucho alimentos diferentes, y caminas tu perro en la playa.   \n",
       "918                                    Entonces, Hawaii es la casa a muchos frutas tropicales, plants unicos, y animales que esta en la isla solamente.   \n",
       "940                                                        Es importante que las gentes de multiples culturas del mundo pueden sentirses representados.   \n",
       "992                                          El cuarto sería una suite con un baño muy grande, una cama cómoda, y una sala con un televisión muy nuevo.   \n",
       "1028  Durante mi vacación perfecto no hay nada para hacer o nada para mi a preocupada a, especialmente tarea o estudiado, solamente necesito relajarme.   \n",
       "\n",
       "                                                                                                                                                                                               annotated_sentence  \\\n",
       "0                                                                                                            No confundas []{a}<az:do:an> Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1                                                                                                  Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}<az:do:an> Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                                                                                                                   Yo miro []{a}<az:do:an> él en la television.   \n",
       "13                                                                                                                                     Yo quiero conocer []{a}<az:do:an> Taylor Swift porque ella no es aburrida.   \n",
       "32                                                                                                            Yo admiro []{a}<az:do:an> Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "...                                                                                                                                                                                                           ...   \n",
       "875                                                           Tu nadas en el mar, [pescadas]{pescados}<ig:f:noun:an>, comes [mucho]{muchos}<na:ps:det:inan> alimentos diferentes, y caminas tu perro en la playa.   \n",
       "918                                            Entonces, Hawaii es la casa a [muchos]{muchas}<ga:fm:det:inan> frutas tropicales, plants [únicos]{únicas}<ga:fm:adj:an>, y animales que esta en la isla solamente.   \n",
       "940                                                            Es importante que las [gentes]{gente}<na:sp:noun:an> de multiples culturas del mundo pueden sentirses [representados]{representada}<ga:fm:adj:an>.   \n",
       "992                                                    El cuarto sería una suite con un baño muy grande, una cama cómoda, y una sala con [un]{una}<ga:fm:det:inan> televisión muy [nuevo]{nueva}<ga:fm:adj:inan>.   \n",
       "1028  Durante mi [vacación]{vacaciones}<in:s:noun:inan> [perfecto]{perfecta}<ga:fm:adj:inan> no hay nada para hacer o nada para mi a preocupada a, especialmente tarea o estudiado, solamente necesito relajarme.   \n",
       "\n",
       "                                   full_error_tag       error_tag  \\\n",
       "0                                 []{a}<az:do:an>        az:do:an   \n",
       "1                                 []{a}<az:do:an>        az:do:an   \n",
       "10                                []{a}<az:do:an>        az:do:an   \n",
       "13                                []{a}<az:do:an>        az:do:an   \n",
       "32                                []{a}<az:do:an>        az:do:an   \n",
       "...                                           ...             ...   \n",
       "875            [pescadas]{pescados}<ig:f:noun:an>    ig:f:noun:an   \n",
       "918                [únicos]{únicas}<ga:fm:adj:an>    ga:fm:adj:an   \n",
       "940   [representados]{representada}<ga:fm:adj:an>    ga:fm:adj:an   \n",
       "992                [nuevo]{nueva}<ga:fm:adj:inan>  ga:fm:adj:inan   \n",
       "1028       [vacación]{vacaciones}<in:s:noun:inan>  in:s:noun:inan   \n",
       "\n",
       "        correction          error  \\\n",
       "0                a            NaN   \n",
       "1                a            NaN   \n",
       "10               a            NaN   \n",
       "13               a            NaN   \n",
       "32               a            NaN   \n",
       "...            ...            ...   \n",
       "875       pescados       pescadas   \n",
       "918         únicas         únicos   \n",
       "940   representada  representados   \n",
       "992          nueva          nuevo   \n",
       "1028    vacaciones       vacación   \n",
       "\n",
       "                                                                                                                                                                                 target  \\\n",
       "0                                                                             [No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1                                                              [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                                                                             [Yo, miro, a, él, en, la, television, .]   \n",
       "13                                                                                                           [Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                                                                               [Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "...                                                                                                                                                                                 ...   \n",
       "875                                                          [Tu, nadas, en, el, mar, ,, pescados, ,, comes, muchos, alimentos, diferentes, ,, y, caminas, tu, perro, en, la, playa, .]   \n",
       "918                                        [Entonces, ,, Hawaii, es, la, casa, a, muchas, frutas, tropicales, ,, plants, únicas, ,, y, animales, que, esta, en, la, isla, solamente, .]   \n",
       "940                                                                          [Es, importante, que, las, gente, de, multiples, culturas, del, mundo, pueden, sentirses, representada, .]   \n",
       "992                                             [El, cuarto, sería, una, suite, con, un, baño, muy, grande, ,, una, cama, cómoda, ,, y, una, sala, con, una, televisión, muy, nueva, .]   \n",
       "1028  [Durante, mi, vacaciones, perfecta, no, hay, nada, para, hacer, o, nada, para, mi, a, preocupada, a, ,, especialmente, tarea, o, estudiado, ,, solamente, necesito, relajarme, .]   \n",
       "\n",
       "                                                                                                                                                                               source  \\\n",
       "0                                                                      [No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1                                                       [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                                                                      [Yo, miro, [NONE], él, en, la, television, .]   \n",
       "13                                                                                                    [Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                                                                        [Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "...                                                                                                                                                                               ...   \n",
       "875                                                        [Tu, nadas, en, el, mar, ,, pescadas, ,, comes, muchos, alimentos, diferentes, ,, y, caminas, tu, perro, en, la, playa, .]   \n",
       "918                                      [Entonces, ,, Hawaii, es, la, casa, a, muchas, frutas, tropicales, ,, plants, únicos, ,, y, animales, que, esta, en, la, isla, solamente, .]   \n",
       "940                                                                       [Es, importante, que, las, gente, de, multiples, culturas, del, mundo, pueden, sentirses, representados, .]   \n",
       "992                                           [El, cuarto, sería, una, suite, con, un, baño, muy, grande, ,, una, cama, cómoda, ,, y, una, sala, con, una, televisión, muy, nuevo, .]   \n",
       "1028  [Durante, mi, vacación, perfecta, no, hay, nada, para, hacer, o, nada, para, mi, a, preocupada, a, ,, especialmente, tarea, o, estudiado, ,, solamente, necesito, relajarme, .]   \n",
       "\n",
       "     correction_index    new_error_type  \\\n",
       "0             [2, 19]           article   \n",
       "1            [11, 32]           article   \n",
       "10            [2, 11]           article   \n",
       "13            [3, 16]           article   \n",
       "32            [2, 18]           article   \n",
       "...               ...               ...   \n",
       "875           [6, 28]  gender agreement   \n",
       "918          [12, 36]  gender agreement   \n",
       "940          [12, 27]  gender agreement   \n",
       "992          [22, 47]  gender agreement   \n",
       "1028          [2, 29]  number agreement   \n",
       "\n",
       "                                                                                                      raw_evidence  \\\n",
       "0     evidence words = \"confundas\", \"[NONE]\", \"Selena Gomez\", \"con\", \"Selena Quintanilla\"\\nindexes = 1, 2, 3, 4, 5   \n",
       "1                               evidence words = Nicolas Cage, ayudo, Johnny Depp, empezar\\nindexes = 7, 9, 11, 12   \n",
       "10                                  evidence words = Yo, miro, él, en, la, television \\nindexes = 0, 1, 3, 4, 5, 6   \n",
       "13                                                      evidence words = \"conocer\", \"Taylor Swift\"\\nindexes = 2, 4   \n",
       "32                                                        evidence words = admiro, Ian Somerhalder\\nindexes = 1, 3   \n",
       "...                                                                                                            ...   \n",
       "875                                                       evidence words = \"mar , pescadas ,\", \"indexes = 4, 5, 6\"   \n",
       "918                                                   evidence words = \", plants únicos ,\", \"indexes = 13, 14, 15\"   \n",
       "940                                                  evidence words = \"la gente\", \"representados\"\\nindexes = 3, 11   \n",
       "992                                                 evidence words = televisión, muy, nuevo \\nindexes = 23, 24, 25   \n",
       "1028                                                            evidence words = \"mi\", \"vacación\" \\nindexes = 1, 2   \n",
       "\n",
       "                                                   evident_words  \\\n",
       "0     [confundas, NONE, Selena, Gomez, con, Selena, Quintanilla]   \n",
       "1                  [Nicolas, Cage, ayudo, Johnny, Depp, empezar]   \n",
       "10                          [Yo, miro, él, en, la, television, ]   \n",
       "13                                      [conocer, Taylor, Swift]   \n",
       "32                                    [admiro, Ian, Somerhalder]   \n",
       "...                                                          ...   \n",
       "875                                          [mar, , pescadas, ]   \n",
       "918                                         [, plants, únicos, ]   \n",
       "940                                   [la, gente, representados]   \n",
       "992                                   [televisión, muy, nuevo, ]   \n",
       "1028                                            [mi, vacación, ]   \n",
       "\n",
       "           evident_index  \n",
       "0        [1, 2, 3, 4, 5]  \n",
       "1         [7, 9, 11, 12]  \n",
       "10    [0, 1, 3, 4, 5, 6]  \n",
       "13                [2, 4]  \n",
       "32                [1, 3]  \n",
       "...                  ...  \n",
       "875            [4, 5, 6]  \n",
       "918         [13, 14, 15]  \n",
       "940              [3, 11]  \n",
       "992         [23, 24, 25]  \n",
       "1028              [1, 2]  \n",
       "\n",
       "[241 rows x 16 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_incorrect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "O68jtHppbx5t",
    "outputId": "276ddbb8-5207-4b55-c7e8-60e292ad836f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_evidence_index_from_list(['mi', 'vacación',''], 'Durante mi vacación perfecta no hay nada para hacer o nada para mi a preocupada a , especialmente tarea o estudiado , solamente necesito relajarme .\t[mi, vacación, ]', [2,29])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qWIPjAHWpiLy",
    "outputId": "a89f94cc-2b29-47d9-ec09-ccad177e93aa"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-74-398dee674024>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_incorrect['evident_index'] = df_incorrect.apply(lambda row : find_evidence_index_from_list(row['evident_words'], row['source'],row['correction_index']), axis=1)\n"
     ]
    }
   ],
   "source": [
    "df_incorrect['evident_index'] = df_incorrect.apply(lambda row : find_evidence_index_from_list(row['evident_words'], row['source'],row['correction_index']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "id": "RU1TIWj6Eoey"
   },
   "outputs": [],
   "source": [
    "df_final_v1 = pd.concat([df_incorrect, df_correct ], axis=0)\n",
    "# to do: if correction_index is in evidence index we have to remove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "ZZTn7igDp-ow"
   },
   "outputs": [],
   "source": [
    "df_final_v1['evident_index_new'] = df_final_v1.apply(lambda row : remove_correction_index(row['correction_index'], row['evident_index']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "id": "uaHV3lS2qVMF"
   },
   "outputs": [],
   "source": [
    "df_final_v1['final_evident_index'] = df_final_v1.apply(lambda row : final_evidence_index(row['evident_index'], row['source']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "id": "Zu7gst9vqk38"
   },
   "outputs": [],
   "source": [
    "df_with_evident = df_final_v1[['paragraph_id','sentence_id', 'course','essay_new', 'annotated_sentence', 'full_error_tag','error_tag','correction','error','target','source', 'correction_index', 'new_error_type','final_evident_index']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "UlnS5gHMgMDo",
    "outputId": "363924f0-e4fa-45a2-a9d9-9a48bace604b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-e988a5e7-33ed-4c04-a83e-0b964682b616\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>source</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>new_error_type</th>\n",
       "      <th>final_evident_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[2, 19]</td>\n",
       "      <td>article</td>\n",
       "      <td>[1, 3, 4, 5, 6, 7, 18, 20, 21, 22, 23, 24]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}&lt;az:do:an&gt; Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[11, 32]</td>\n",
       "      <td>article</td>\n",
       "      <td>[7, 8, 10, 12, 13, 14, 28, 29, 31, 33, 34, 35]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>71</td>\n",
       "      <td>32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Yo miro él en la television.</td>\n",
       "      <td>Yo miro []{a}&lt;az:do:an&gt; él en la television.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, miro, a, él, en, la, television, .]</td>\n",
       "      <td>[Yo, miro, [NONE], él, en, la, television, .]</td>\n",
       "      <td>[2, 11]</td>\n",
       "      <td>article</td>\n",
       "      <td>[0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 14, 15]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>93</td>\n",
       "      <td>b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo quiero conocer Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>Yo quiero conocer []{a}&lt;az:do:an&gt; Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[3, 16]</td>\n",
       "      <td>article</td>\n",
       "      <td>[2, 4, 5, 15, 17, 18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>139</td>\n",
       "      <td>7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>Yo admiro []{a}&lt;az:do:an&gt; Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[2, 18]</td>\n",
       "      <td>article</td>\n",
       "      <td>[1, 3, 4, 17, 19, 20]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1137</td>\n",
       "      <td>4dc889230c51fb53fa2008f196ac32ef1ae3c8588f4e1ec6323e3a56df482e63</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Para la primera día, espero que comer más comida local.</td>\n",
       "      <td>Para [la]{el}&lt;ga:mf:det:aty:inan&gt; [primera]{primer}&lt;ga:mf:det:aty:inan&gt; día, espero que comer más comida local.</td>\n",
       "      <td>[la]{el}&lt;ga:mf:det:aty:inan&gt;</td>\n",
       "      <td>ga:mf:det:aty:inan</td>\n",
       "      <td>el</td>\n",
       "      <td>la</td>\n",
       "      <td>[Para, el, primer, día, ,, espero, que, comer, más, comida, local, .]</td>\n",
       "      <td>[Para, la, primer, día, ,, espero, que, comer, más, comida, local, .]</td>\n",
       "      <td>[1, 14]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[2, 15]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1205</td>\n",
       "      <td>ddca439dbc58c89c77ee3910845957961a571cdcad0f857007e0486095ae349b</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Nosotros traemos la gata en el resto de la vacaciones.</td>\n",
       "      <td>[Nosotros]{nosotras}&lt;ga:fm:pron:an&gt; traemos la gata en el resto de [la]{las}&lt;na:ps:det:inan&gt; vacaciones.</td>\n",
       "      <td>[Nosotros]{nosotras}&lt;ga:fm:pron:an&gt;</td>\n",
       "      <td>ga:fm:pron:an</td>\n",
       "      <td>nosotras</td>\n",
       "      <td>Nosotros</td>\n",
       "      <td>[nosotras, traemos, la, gata, en, el, resto, de, las, vacaciones, .]</td>\n",
       "      <td>[Nosotros, traemos, la, gata, en, el, resto, de, las, vacaciones, .]</td>\n",
       "      <td>[0, 12]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[3, 15]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1384</td>\n",
       "      <td>c1ca2f5c15fe7682ab4f1503799d0f9f2300b495301aaa2735beefa62f7c64b6</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>El perfecto vacaciones es un vacacion muy reljante y bonito.</td>\n",
       "      <td>[El]{las}&lt;ga:na:fm:ps:det:inan&gt; [perfecto]{perfectas}&lt;ga:na:fm:ps:adj:inan&gt; vacaciones es [un]{una}&lt;ga:fm:det:inan&gt; [vacacion]{vacaciones}&lt;in:s:gat:noun:inan&gt; muy reljante y [bonito]{bonita}&lt;ga:fm:adj:inan&gt;.</td>\n",
       "      <td>[perfecto]{perfectas}&lt;ga:na:fm:ps:adj:inan&gt;</td>\n",
       "      <td>ga:na:fm:ps:adj:inan</td>\n",
       "      <td>perfectas</td>\n",
       "      <td>perfecto</td>\n",
       "      <td>[las, perfectas, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]</td>\n",
       "      <td>[las, perfecto, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]</td>\n",
       "      <td>[1, 13]</td>\n",
       "      <td>gender and number agreement</td>\n",
       "      <td>[0, 12]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1548</td>\n",
       "      <td>35dfbba951e96c672c60bdb33ffd2ad4523ea3a2e55a42404cd1451ac675b447</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Es el mas azul playa en California y mi favorito playa en el mundo.</td>\n",
       "      <td>Es [el]{la}&lt;ga:fm:det:inan&gt; mas azul playa en California y mi [favorito]{favorita}&lt;ga:fm:adj:inan&gt; playa en el mundo.</td>\n",
       "      <td>[favorito]{favorita}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>ga:fm:adj:inan</td>\n",
       "      <td>favorita</td>\n",
       "      <td>favorito</td>\n",
       "      <td>[Es, la, mas, azul, playa, en, California, y, mi, favorita, playa, en, el, mundo, .]</td>\n",
       "      <td>[Es, la, mas, azul, playa, en, California, y, mi, favorito, playa, en, el, mundo, .]</td>\n",
       "      <td>[9, 25]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[10, 11, 26, 27]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1138</td>\n",
       "      <td>86117767a230e4bc9a5f64dbbfc5fd6b08bc8d56670d47fb07c9056687f211f6</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>\\n\\nEn la primera día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "      <td>En [la]{el}&lt;ga:mf:det:aty:inan&gt; [primera]{primer}&lt;ga:mf:det:aty:inan&gt; día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "      <td>[la]{el}&lt;ga:mf:det:aty:inan&gt;</td>\n",
       "      <td>ga:mf:det:aty:inan</td>\n",
       "      <td>el</td>\n",
       "      <td>la</td>\n",
       "      <td>[En, el, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]</td>\n",
       "      <td>[En, la, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]</td>\n",
       "      <td>[1, 22]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[2, 23]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5212 rows × 14 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e988a5e7-33ed-4c04-a83e-0b964682b616')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-e988a5e7-33ed-4c04-a83e-0b964682b616 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-e988a5e7-33ed-4c04-a83e-0b964682b616');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-112b2acc-eedb-4f6e-8c3e-6640834a0adf\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-112b2acc-eedb-4f6e-8c3e-6640834a0adf')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-112b2acc-eedb-4f6e-8c3e-6640834a0adf button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id  \\\n",
       "0             13   \n",
       "1             18   \n",
       "10            71   \n",
       "13            93   \n",
       "32           139   \n",
       "..           ...   \n",
       "43          1137   \n",
       "45          1205   \n",
       "46          1384   \n",
       "47          1548   \n",
       "48          1138   \n",
       "\n",
       "                                                         sentence_id course  \\\n",
       "0   452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e  SPA 1   \n",
       "1   e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b  SPA 2   \n",
       "10  32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6  SPA 1   \n",
       "13  b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2  SPA 2   \n",
       "32  7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f  SPA 2   \n",
       "..                                                               ...    ...   \n",
       "43  4dc889230c51fb53fa2008f196ac32ef1ae3c8588f4e1ec6323e3a56df482e63  SPA 3   \n",
       "45  ddca439dbc58c89c77ee3910845957961a571cdcad0f857007e0486095ae349b  SPA 1   \n",
       "46  c1ca2f5c15fe7682ab4f1503799d0f9f2300b495301aaa2735beefa62f7c64b6  SPA 3   \n",
       "47  35dfbba951e96c672c60bdb33ffd2ad4523ea3a2e55a42404cd1451ac675b447  SPA 2   \n",
       "48  86117767a230e4bc9a5f64dbbfc5fd6b08bc8d56670d47fb07c9056687f211f6  SPA 2   \n",
       "\n",
       "                                                                                                 essay_new  \\\n",
       "0                     No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1           Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                            Yo miro él en la television.   \n",
       "13                                              Yo quiero conocer Taylor Swift porque ella no es aburrida.   \n",
       "32                     Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "..                                                                                                     ...   \n",
       "43                                                 Para la primera día, espero que comer más comida local.   \n",
       "45                                                  Nosotros traemos la gata en el resto de la vacaciones.   \n",
       "46                                            El perfecto vacaciones es un vacacion muy reljante y bonito.   \n",
       "47                                     Es el mas azul playa en California y mi favorito playa en el mundo.   \n",
       "48  \\n\\nEn la primera día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.   \n",
       "\n",
       "                                                                                                                                                                                                 annotated_sentence  \\\n",
       "0                                                                                                              No confundas []{a}<az:do:an> Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1                                                                                                    Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}<az:do:an> Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                                                                                                                     Yo miro []{a}<az:do:an> él en la television.   \n",
       "13                                                                                                                                       Yo quiero conocer []{a}<az:do:an> Taylor Swift porque ella no es aburrida.   \n",
       "32                                                                                                              Yo admiro []{a}<az:do:an> Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "..                                                                                                                                                                                                              ...   \n",
       "43                                                                                                  Para [la]{el}<ga:mf:det:aty:inan> [primera]{primer}<ga:mf:det:aty:inan> día, espero que comer más comida local.   \n",
       "45                                                                                                         [Nosotros]{nosotras}<ga:fm:pron:an> traemos la gata en el resto de [la]{las}<na:ps:det:inan> vacaciones.   \n",
       "46  [El]{las}<ga:na:fm:ps:det:inan> [perfecto]{perfectas}<ga:na:fm:ps:adj:inan> vacaciones es [un]{una}<ga:fm:det:inan> [vacacion]{vacaciones}<in:s:gat:noun:inan> muy reljante y [bonito]{bonita}<ga:fm:adj:inan>.   \n",
       "47                                                                                            Es [el]{la}<ga:fm:det:inan> mas azul playa en California y mi [favorito]{favorita}<ga:fm:adj:inan> playa en el mundo.   \n",
       "48                                                       En [la]{el}<ga:mf:det:aty:inan> [primera]{primer}<ga:mf:det:aty:inan> día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.   \n",
       "\n",
       "                                 full_error_tag             error_tag  \\\n",
       "0                               []{a}<az:do:an>              az:do:an   \n",
       "1                               []{a}<az:do:an>              az:do:an   \n",
       "10                              []{a}<az:do:an>              az:do:an   \n",
       "13                              []{a}<az:do:an>              az:do:an   \n",
       "32                              []{a}<az:do:an>              az:do:an   \n",
       "..                                          ...                   ...   \n",
       "43                 [la]{el}<ga:mf:det:aty:inan>    ga:mf:det:aty:inan   \n",
       "45          [Nosotros]{nosotras}<ga:fm:pron:an>         ga:fm:pron:an   \n",
       "46  [perfecto]{perfectas}<ga:na:fm:ps:adj:inan>  ga:na:fm:ps:adj:inan   \n",
       "47         [favorito]{favorita}<ga:fm:adj:inan>        ga:fm:adj:inan   \n",
       "48                 [la]{el}<ga:mf:det:aty:inan>    ga:mf:det:aty:inan   \n",
       "\n",
       "   correction     error  \\\n",
       "0           a       NaN   \n",
       "1           a       NaN   \n",
       "10          a       NaN   \n",
       "13          a       NaN   \n",
       "32          a       NaN   \n",
       "..        ...       ...   \n",
       "43         el        la   \n",
       "45   nosotras  Nosotros   \n",
       "46  perfectas  perfecto   \n",
       "47   favorita  favorito   \n",
       "48         el        la   \n",
       "\n",
       "                                                                                                                      target  \\\n",
       "0                  [No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1   [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                  [Yo, miro, a, él, en, la, television, .]   \n",
       "13                                                [Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                    [Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "..                                                                                                                       ...   \n",
       "43                                                     [Para, el, primer, día, ,, espero, que, comer, más, comida, local, .]   \n",
       "45                                                      [nosotras, traemos, la, gata, en, el, resto, de, las, vacaciones, .]   \n",
       "46                                            [las, perfectas, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]   \n",
       "47                                      [Es, la, mas, azul, playa, en, California, y, mi, favorita, playa, en, el, mundo, .]   \n",
       "48  [En, el, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]   \n",
       "\n",
       "                                                                                                                           source  \\\n",
       "0                  [No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1   [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                  [Yo, miro, [NONE], él, en, la, television, .]   \n",
       "13                                                [Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                    [Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "..                                                                                                                            ...   \n",
       "43                                                          [Para, la, primer, día, ,, espero, que, comer, más, comida, local, .]   \n",
       "45                                                           [Nosotros, traemos, la, gata, en, el, resto, de, las, vacaciones, .]   \n",
       "46                                                  [las, perfecto, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]   \n",
       "47                                           [Es, la, mas, azul, playa, en, California, y, mi, favorito, playa, en, el, mundo, .]   \n",
       "48       [En, la, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]   \n",
       "\n",
       "   correction_index               new_error_type  \\\n",
       "0           [2, 19]                      article   \n",
       "1          [11, 32]                      article   \n",
       "10          [2, 11]                      article   \n",
       "13          [3, 16]                      article   \n",
       "32          [2, 18]                      article   \n",
       "..              ...                          ...   \n",
       "43          [1, 14]             gender agreement   \n",
       "45          [0, 12]             gender agreement   \n",
       "46          [1, 13]  gender and number agreement   \n",
       "47          [9, 25]             gender agreement   \n",
       "48          [1, 22]             gender agreement   \n",
       "\n",
       "                               final_evident_index  \n",
       "0       [1, 3, 4, 5, 6, 7, 18, 20, 21, 22, 23, 24]  \n",
       "1   [7, 8, 10, 12, 13, 14, 28, 29, 31, 33, 34, 35]  \n",
       "10       [0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 14, 15]  \n",
       "13                           [2, 4, 5, 15, 17, 18]  \n",
       "32                           [1, 3, 4, 17, 19, 20]  \n",
       "..                                             ...  \n",
       "43                                         [2, 15]  \n",
       "45                                         [3, 15]  \n",
       "46                                         [0, 12]  \n",
       "47                                [10, 11, 26, 27]  \n",
       "48                                         [2, 23]  \n",
       "\n",
       "[5212 rows x 14 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_evident"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "id": "_WbH4Caqnw9c"
   },
   "outputs": [],
   "source": [
    "df_with_evident.to_csv('dataset_with_evident_word.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "id": "6OjLK8kjB-Zq"
   },
   "outputs": [],
   "source": [
    "# put the source in list into a sentence\n",
    "import string\n",
    "\n",
    "def new_joiner(list_input):\n",
    "  quotation = 0\n",
    "  asteris = 0\n",
    "  strings = ''\n",
    "  for i in range(0, len(list_input)-1):\n",
    "    if list_input[i][0] not in string.punctuation:\n",
    "      if list_input[i+1] == '\"':\n",
    "         if quotation %2 == 0:\n",
    "              strings += list_input[i]\n",
    "              strings += ' '\n",
    "         else:\n",
    "              strings += list_input[i]\n",
    "        #  quotation =+ 1\n",
    "      elif list_input[i+1] == '*':\n",
    "         if asteris %2 == 0:\n",
    "              strings += list_input[i]\n",
    "              strings += ' '\n",
    "         else:\n",
    "              strings += list_input[i]\n",
    "        #  asteris =+ 1\n",
    "      elif list_input[i+1] in string.punctuation:\n",
    "         strings += list_input[i]\n",
    "      else:\n",
    "          strings += list_input[i]\n",
    "          strings += ' '\n",
    "\n",
    "    elif list_input[i][0] in string.punctuation:\n",
    "      if list_input[i+1] == '\"':\n",
    "        if quotation %2 == 0:\n",
    "          strings += list_input[i]\n",
    "          strings += ' '\n",
    "        else:\n",
    "          strings += list_input[i]\n",
    "        # quotation =+ 1\n",
    "      elif list_input[i+1] == '*':\n",
    "          if asteris %2 == 0:\n",
    "            strings += list_input[i]\n",
    "          else:\n",
    "            strings += list_input[i]\n",
    "            strings += ' '\n",
    "          # asteris += 1\n",
    "      else:\n",
    "        if list_input[i] == '\"':\n",
    "          if quotation %2 == 0:\n",
    "            strings += list_input[i]\n",
    "          else:\n",
    "            strings += list_input[i]\n",
    "            strings += ' '\n",
    "          quotation =+ 1\n",
    "        elif list_input[i] == '*':\n",
    "          if asteris %2 == 0:\n",
    "            strings += list_input[i]\n",
    "          else:\n",
    "            strings += list_input[i]\n",
    "            strings += ' '\n",
    "          asteris += 1\n",
    "        else:\n",
    "          strings += list_input[i]\n",
    "          strings += ' '\n",
    "\n",
    "  return strings +list_input[-1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "id": "7jMSjjmlrdY1"
   },
   "outputs": [],
   "source": [
    "# a = ['Mi', 'visité', '*', 'FIRST_NAME', '*', 'en', 'la', 'verano', '.']\n",
    "# b = [ 'reciente', ',', '\"', 'LEMONADE', '.', '\"']\n",
    "\n",
    "c = ['No', 'confundas', 'a', 'Selena', 'Gomez', 'con', 'Selena', 'Quintanilla', ',', 'de', 'quien', 'Selena', 'Gomez', 'es', 'nombrada', '.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "fyqWi2km6ULM",
    "outputId": "e0d29eee-5827-4277-b8c2-adeae9f29ca0"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'Mi visité *FIRST_NAME* en la verano.'"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_joiner(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "QeMbP681shH0",
    "outputId": "fd4fda7e-f119-4398-d638-faa3d197a939"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'No confundas a Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.'"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_joiner(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "24-4LhNZta12",
    "outputId": "c77c97f2-f5d0-4a9d-fbab-e7d2c3fa67cc"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'reciente, \"LEMONADE.\"'"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_joiner(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "Gp1jMgiwrEgQ",
    "outputId": "b0bfb851-59ae-405d-8ea1-4dbb03598c7a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-082ea249-db28-4939-b749-86248b73fcf2\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>source</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>new_error_type</th>\n",
       "      <th>final_evident_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[2, 19]</td>\n",
       "      <td>article</td>\n",
       "      <td>[1, 3, 4, 5, 6, 7, 18, 20, 21, 22, 23, 24]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}&lt;az:do:an&gt; Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[11, 32]</td>\n",
       "      <td>article</td>\n",
       "      <td>[7, 8, 10, 12, 13, 14, 28, 29, 31, 33, 34, 35]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>71</td>\n",
       "      <td>32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Yo miro él en la television.</td>\n",
       "      <td>Yo miro []{a}&lt;az:do:an&gt; él en la television.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, miro, a, él, en, la, television, .]</td>\n",
       "      <td>[Yo, miro, [NONE], él, en, la, television, .]</td>\n",
       "      <td>[2, 11]</td>\n",
       "      <td>article</td>\n",
       "      <td>[0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 14, 15]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>93</td>\n",
       "      <td>b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo quiero conocer Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>Yo quiero conocer []{a}&lt;az:do:an&gt; Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[3, 16]</td>\n",
       "      <td>article</td>\n",
       "      <td>[2, 4, 5, 15, 17, 18]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>139</td>\n",
       "      <td>7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>Yo admiro []{a}&lt;az:do:an&gt; Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[2, 18]</td>\n",
       "      <td>article</td>\n",
       "      <td>[1, 3, 4, 17, 19, 20]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1137</td>\n",
       "      <td>4dc889230c51fb53fa2008f196ac32ef1ae3c8588f4e1ec6323e3a56df482e63</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Para la primera día, espero que comer más comida local.</td>\n",
       "      <td>Para [la]{el}&lt;ga:mf:det:aty:inan&gt; [primera]{primer}&lt;ga:mf:det:aty:inan&gt; día, espero que comer más comida local.</td>\n",
       "      <td>[la]{el}&lt;ga:mf:det:aty:inan&gt;</td>\n",
       "      <td>ga:mf:det:aty:inan</td>\n",
       "      <td>el</td>\n",
       "      <td>la</td>\n",
       "      <td>[Para, el, primer, día, ,, espero, que, comer, más, comida, local, .]</td>\n",
       "      <td>[Para, la, primer, día, ,, espero, que, comer, más, comida, local, .]</td>\n",
       "      <td>[1, 14]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[2, 15]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1205</td>\n",
       "      <td>ddca439dbc58c89c77ee3910845957961a571cdcad0f857007e0486095ae349b</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Nosotros traemos la gata en el resto de la vacaciones.</td>\n",
       "      <td>[Nosotros]{nosotras}&lt;ga:fm:pron:an&gt; traemos la gata en el resto de [la]{las}&lt;na:ps:det:inan&gt; vacaciones.</td>\n",
       "      <td>[Nosotros]{nosotras}&lt;ga:fm:pron:an&gt;</td>\n",
       "      <td>ga:fm:pron:an</td>\n",
       "      <td>nosotras</td>\n",
       "      <td>Nosotros</td>\n",
       "      <td>[nosotras, traemos, la, gata, en, el, resto, de, las, vacaciones, .]</td>\n",
       "      <td>[Nosotros, traemos, la, gata, en, el, resto, de, las, vacaciones, .]</td>\n",
       "      <td>[0, 12]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[3, 15]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1384</td>\n",
       "      <td>c1ca2f5c15fe7682ab4f1503799d0f9f2300b495301aaa2735beefa62f7c64b6</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>El perfecto vacaciones es un vacacion muy reljante y bonito.</td>\n",
       "      <td>[El]{las}&lt;ga:na:fm:ps:det:inan&gt; [perfecto]{perfectas}&lt;ga:na:fm:ps:adj:inan&gt; vacaciones es [un]{una}&lt;ga:fm:det:inan&gt; [vacacion]{vacaciones}&lt;in:s:gat:noun:inan&gt; muy reljante y [bonito]{bonita}&lt;ga:fm:adj:inan&gt;.</td>\n",
       "      <td>[perfecto]{perfectas}&lt;ga:na:fm:ps:adj:inan&gt;</td>\n",
       "      <td>ga:na:fm:ps:adj:inan</td>\n",
       "      <td>perfectas</td>\n",
       "      <td>perfecto</td>\n",
       "      <td>[las, perfectas, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]</td>\n",
       "      <td>[las, perfecto, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]</td>\n",
       "      <td>[1, 13]</td>\n",
       "      <td>gender and number agreement</td>\n",
       "      <td>[0, 12]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1548</td>\n",
       "      <td>35dfbba951e96c672c60bdb33ffd2ad4523ea3a2e55a42404cd1451ac675b447</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Es el mas azul playa en California y mi favorito playa en el mundo.</td>\n",
       "      <td>Es [el]{la}&lt;ga:fm:det:inan&gt; mas azul playa en California y mi [favorito]{favorita}&lt;ga:fm:adj:inan&gt; playa en el mundo.</td>\n",
       "      <td>[favorito]{favorita}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>ga:fm:adj:inan</td>\n",
       "      <td>favorita</td>\n",
       "      <td>favorito</td>\n",
       "      <td>[Es, la, mas, azul, playa, en, California, y, mi, favorita, playa, en, el, mundo, .]</td>\n",
       "      <td>[Es, la, mas, azul, playa, en, California, y, mi, favorito, playa, en, el, mundo, .]</td>\n",
       "      <td>[9, 25]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[10, 11, 26, 27]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1138</td>\n",
       "      <td>86117767a230e4bc9a5f64dbbfc5fd6b08bc8d56670d47fb07c9056687f211f6</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>\\n\\nEn la primera día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "      <td>En [la]{el}&lt;ga:mf:det:aty:inan&gt; [primera]{primer}&lt;ga:mf:det:aty:inan&gt; día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "      <td>[la]{el}&lt;ga:mf:det:aty:inan&gt;</td>\n",
       "      <td>ga:mf:det:aty:inan</td>\n",
       "      <td>el</td>\n",
       "      <td>la</td>\n",
       "      <td>[En, el, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]</td>\n",
       "      <td>[En, la, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]</td>\n",
       "      <td>[1, 22]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[2, 23]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5212 rows × 14 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-082ea249-db28-4939-b749-86248b73fcf2')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-082ea249-db28-4939-b749-86248b73fcf2 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-082ea249-db28-4939-b749-86248b73fcf2');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-c10205b2-1684-4ad6-865a-4b3cccd167e7\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c10205b2-1684-4ad6-865a-4b3cccd167e7')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-c10205b2-1684-4ad6-865a-4b3cccd167e7 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id  \\\n",
       "0             13   \n",
       "1             18   \n",
       "10            71   \n",
       "13            93   \n",
       "32           139   \n",
       "..           ...   \n",
       "43          1137   \n",
       "45          1205   \n",
       "46          1384   \n",
       "47          1548   \n",
       "48          1138   \n",
       "\n",
       "                                                         sentence_id course  \\\n",
       "0   452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e  SPA 1   \n",
       "1   e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b  SPA 2   \n",
       "10  32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6  SPA 1   \n",
       "13  b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2  SPA 2   \n",
       "32  7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f  SPA 2   \n",
       "..                                                               ...    ...   \n",
       "43  4dc889230c51fb53fa2008f196ac32ef1ae3c8588f4e1ec6323e3a56df482e63  SPA 3   \n",
       "45  ddca439dbc58c89c77ee3910845957961a571cdcad0f857007e0486095ae349b  SPA 1   \n",
       "46  c1ca2f5c15fe7682ab4f1503799d0f9f2300b495301aaa2735beefa62f7c64b6  SPA 3   \n",
       "47  35dfbba951e96c672c60bdb33ffd2ad4523ea3a2e55a42404cd1451ac675b447  SPA 2   \n",
       "48  86117767a230e4bc9a5f64dbbfc5fd6b08bc8d56670d47fb07c9056687f211f6  SPA 2   \n",
       "\n",
       "                                                                                                 essay_new  \\\n",
       "0                     No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1           Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                            Yo miro él en la television.   \n",
       "13                                              Yo quiero conocer Taylor Swift porque ella no es aburrida.   \n",
       "32                     Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "..                                                                                                     ...   \n",
       "43                                                 Para la primera día, espero que comer más comida local.   \n",
       "45                                                  Nosotros traemos la gata en el resto de la vacaciones.   \n",
       "46                                            El perfecto vacaciones es un vacacion muy reljante y bonito.   \n",
       "47                                     Es el mas azul playa en California y mi favorito playa en el mundo.   \n",
       "48  \\n\\nEn la primera día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.   \n",
       "\n",
       "                                                                                                                                                                                                 annotated_sentence  \\\n",
       "0                                                                                                              No confundas []{a}<az:do:an> Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1                                                                                                    Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}<az:do:an> Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                                                                                                                     Yo miro []{a}<az:do:an> él en la television.   \n",
       "13                                                                                                                                       Yo quiero conocer []{a}<az:do:an> Taylor Swift porque ella no es aburrida.   \n",
       "32                                                                                                              Yo admiro []{a}<az:do:an> Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "..                                                                                                                                                                                                              ...   \n",
       "43                                                                                                  Para [la]{el}<ga:mf:det:aty:inan> [primera]{primer}<ga:mf:det:aty:inan> día, espero que comer más comida local.   \n",
       "45                                                                                                         [Nosotros]{nosotras}<ga:fm:pron:an> traemos la gata en el resto de [la]{las}<na:ps:det:inan> vacaciones.   \n",
       "46  [El]{las}<ga:na:fm:ps:det:inan> [perfecto]{perfectas}<ga:na:fm:ps:adj:inan> vacaciones es [un]{una}<ga:fm:det:inan> [vacacion]{vacaciones}<in:s:gat:noun:inan> muy reljante y [bonito]{bonita}<ga:fm:adj:inan>.   \n",
       "47                                                                                            Es [el]{la}<ga:fm:det:inan> mas azul playa en California y mi [favorito]{favorita}<ga:fm:adj:inan> playa en el mundo.   \n",
       "48                                                       En [la]{el}<ga:mf:det:aty:inan> [primera]{primer}<ga:mf:det:aty:inan> día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.   \n",
       "\n",
       "                                 full_error_tag             error_tag  \\\n",
       "0                               []{a}<az:do:an>              az:do:an   \n",
       "1                               []{a}<az:do:an>              az:do:an   \n",
       "10                              []{a}<az:do:an>              az:do:an   \n",
       "13                              []{a}<az:do:an>              az:do:an   \n",
       "32                              []{a}<az:do:an>              az:do:an   \n",
       "..                                          ...                   ...   \n",
       "43                 [la]{el}<ga:mf:det:aty:inan>    ga:mf:det:aty:inan   \n",
       "45          [Nosotros]{nosotras}<ga:fm:pron:an>         ga:fm:pron:an   \n",
       "46  [perfecto]{perfectas}<ga:na:fm:ps:adj:inan>  ga:na:fm:ps:adj:inan   \n",
       "47         [favorito]{favorita}<ga:fm:adj:inan>        ga:fm:adj:inan   \n",
       "48                 [la]{el}<ga:mf:det:aty:inan>    ga:mf:det:aty:inan   \n",
       "\n",
       "   correction     error  \\\n",
       "0           a       NaN   \n",
       "1           a       NaN   \n",
       "10          a       NaN   \n",
       "13          a       NaN   \n",
       "32          a       NaN   \n",
       "..        ...       ...   \n",
       "43         el        la   \n",
       "45   nosotras  Nosotros   \n",
       "46  perfectas  perfecto   \n",
       "47   favorita  favorito   \n",
       "48         el        la   \n",
       "\n",
       "                                                                                                                      target  \\\n",
       "0                  [No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1   [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                  [Yo, miro, a, él, en, la, television, .]   \n",
       "13                                                [Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                    [Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "..                                                                                                                       ...   \n",
       "43                                                     [Para, el, primer, día, ,, espero, que, comer, más, comida, local, .]   \n",
       "45                                                      [nosotras, traemos, la, gata, en, el, resto, de, las, vacaciones, .]   \n",
       "46                                            [las, perfectas, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]   \n",
       "47                                      [Es, la, mas, azul, playa, en, California, y, mi, favorita, playa, en, el, mundo, .]   \n",
       "48  [En, el, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]   \n",
       "\n",
       "                                                                                                                           source  \\\n",
       "0                  [No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1   [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                  [Yo, miro, [NONE], él, en, la, television, .]   \n",
       "13                                                [Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                    [Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "..                                                                                                                            ...   \n",
       "43                                                          [Para, la, primer, día, ,, espero, que, comer, más, comida, local, .]   \n",
       "45                                                           [Nosotros, traemos, la, gata, en, el, resto, de, las, vacaciones, .]   \n",
       "46                                                  [las, perfecto, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]   \n",
       "47                                           [Es, la, mas, azul, playa, en, California, y, mi, favorito, playa, en, el, mundo, .]   \n",
       "48       [En, la, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]   \n",
       "\n",
       "   correction_index               new_error_type  \\\n",
       "0           [2, 19]                      article   \n",
       "1          [11, 32]                      article   \n",
       "10          [2, 11]                      article   \n",
       "13          [3, 16]                      article   \n",
       "32          [2, 18]                      article   \n",
       "..              ...                          ...   \n",
       "43          [1, 14]             gender agreement   \n",
       "45          [0, 12]             gender agreement   \n",
       "46          [1, 13]  gender and number agreement   \n",
       "47          [9, 25]             gender agreement   \n",
       "48          [1, 22]             gender agreement   \n",
       "\n",
       "                               final_evident_index  \n",
       "0       [1, 3, 4, 5, 6, 7, 18, 20, 21, 22, 23, 24]  \n",
       "1   [7, 8, 10, 12, 13, 14, 28, 29, 31, 33, 34, 35]  \n",
       "10       [0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 14, 15]  \n",
       "13                           [2, 4, 5, 15, 17, 18]  \n",
       "32                           [1, 3, 4, 17, 19, 20]  \n",
       "..                                             ...  \n",
       "43                                         [2, 15]  \n",
       "45                                         [3, 15]  \n",
       "46                                         [0, 12]  \n",
       "47                                [10, 11, 26, 27]  \n",
       "48                                         [2, 23]  \n",
       "\n",
       "[5212 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_evident"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "leiD9dc9q8_m",
    "outputId": "64e8bc2b-fcb4-440c-dffb-a998010bfbf3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-225-0fd8e28e34e4>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_with_evident['target_sentence'] = df_with_evident['target'].apply(new_joiner)\n",
      "<ipython-input-225-0fd8e28e34e4>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_with_evident['source_sentence'] = df_with_evident['source'].apply(new_joiner)\n"
     ]
    }
   ],
   "source": [
    "df_with_evident['target_sentence'] = df_with_evident['target'].apply(new_joiner)\n",
    "df_with_evident['source_sentence'] = df_with_evident['source'].apply(new_joiner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "FjycLzXZ5LRQ",
    "outputId": "fdb577e4-771b-464c-95f7-76dead3f240f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-0a28d341-023f-4558-9383-533cf01987b2\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>source</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>new_error_type</th>\n",
       "      <th>final_evident_index</th>\n",
       "      <th>target_sentence</th>\n",
       "      <th>source_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13</td>\n",
       "      <td>452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>No confundas []{a}&lt;az:do:an&gt; Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]</td>\n",
       "      <td>[2, 19]</td>\n",
       "      <td>article</td>\n",
       "      <td>[1, 3, 4, 5, 6, 7, 18, 20, 21, 22, 23, 24]</td>\n",
       "      <td>No confundas a Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "      <td>No confundas [NONE] Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}&lt;az:do:an&gt; Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]</td>\n",
       "      <td>[11, 32]</td>\n",
       "      <td>article</td>\n",
       "      <td>[7, 8, 10, 12, 13, 14, 28, 29, 31, 33, 34, 35]</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo a Johnny Depp empezar su profesión de actor.</td>\n",
       "      <td>Un Otro famoso actor, que llama Nicolas Cage, ayudo [NONE] Johnny Depp empezar su profesión de actor.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>71</td>\n",
       "      <td>32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Yo miro él en la television.</td>\n",
       "      <td>Yo miro []{a}&lt;az:do:an&gt; él en la television.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, miro, a, él, en, la, television, .]</td>\n",
       "      <td>[Yo, miro, [NONE], él, en, la, television, .]</td>\n",
       "      <td>[2, 11]</td>\n",
       "      <td>article</td>\n",
       "      <td>[0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 14, 15]</td>\n",
       "      <td>Yo miro a él en la television.</td>\n",
       "      <td>Yo miro [NONE] él en la television.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>93</td>\n",
       "      <td>b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo quiero conocer Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>Yo quiero conocer []{a}&lt;az:do:an&gt; Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]</td>\n",
       "      <td>[3, 16]</td>\n",
       "      <td>article</td>\n",
       "      <td>[2, 4, 5, 15, 17, 18]</td>\n",
       "      <td>Yo quiero conocer a Taylor Swift porque ella no es aburrida.</td>\n",
       "      <td>Yo quiero conocer [NONE] Taylor Swift porque ella no es aburrida.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>139</td>\n",
       "      <td>7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>Yo admiro []{a}&lt;az:do:an&gt; Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>[]{a}&lt;az:do:an&gt;</td>\n",
       "      <td>az:do:an</td>\n",
       "      <td>a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]</td>\n",
       "      <td>[2, 18]</td>\n",
       "      <td>article</td>\n",
       "      <td>[1, 3, 4, 17, 19, 20]</td>\n",
       "      <td>Yo admiro a Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "      <td>Yo admiro [NONE] Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1137</td>\n",
       "      <td>4dc889230c51fb53fa2008f196ac32ef1ae3c8588f4e1ec6323e3a56df482e63</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>Para la primera día, espero que comer más comida local.</td>\n",
       "      <td>Para [la]{el}&lt;ga:mf:det:aty:inan&gt; [primera]{primer}&lt;ga:mf:det:aty:inan&gt; día, espero que comer más comida local.</td>\n",
       "      <td>[la]{el}&lt;ga:mf:det:aty:inan&gt;</td>\n",
       "      <td>ga:mf:det:aty:inan</td>\n",
       "      <td>el</td>\n",
       "      <td>la</td>\n",
       "      <td>[Para, el, primer, día, ,, espero, que, comer, más, comida, local, .]</td>\n",
       "      <td>[Para, la, primer, día, ,, espero, que, comer, más, comida, local, .]</td>\n",
       "      <td>[1, 14]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[2, 15]</td>\n",
       "      <td>Para el primer día, espero que comer más comida local.</td>\n",
       "      <td>Para la primer día, espero que comer más comida local.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1205</td>\n",
       "      <td>ddca439dbc58c89c77ee3910845957961a571cdcad0f857007e0486095ae349b</td>\n",
       "      <td>SPA 1</td>\n",
       "      <td>Nosotros traemos la gata en el resto de la vacaciones.</td>\n",
       "      <td>[Nosotros]{nosotras}&lt;ga:fm:pron:an&gt; traemos la gata en el resto de [la]{las}&lt;na:ps:det:inan&gt; vacaciones.</td>\n",
       "      <td>[Nosotros]{nosotras}&lt;ga:fm:pron:an&gt;</td>\n",
       "      <td>ga:fm:pron:an</td>\n",
       "      <td>nosotras</td>\n",
       "      <td>Nosotros</td>\n",
       "      <td>[nosotras, traemos, la, gata, en, el, resto, de, las, vacaciones, .]</td>\n",
       "      <td>[Nosotros, traemos, la, gata, en, el, resto, de, las, vacaciones, .]</td>\n",
       "      <td>[0, 12]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[3, 15]</td>\n",
       "      <td>nosotras traemos la gata en el resto de las vacaciones.</td>\n",
       "      <td>Nosotros traemos la gata en el resto de las vacaciones.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1384</td>\n",
       "      <td>c1ca2f5c15fe7682ab4f1503799d0f9f2300b495301aaa2735beefa62f7c64b6</td>\n",
       "      <td>SPA 3</td>\n",
       "      <td>El perfecto vacaciones es un vacacion muy reljante y bonito.</td>\n",
       "      <td>[El]{las}&lt;ga:na:fm:ps:det:inan&gt; [perfecto]{perfectas}&lt;ga:na:fm:ps:adj:inan&gt; vacaciones es [un]{una}&lt;ga:fm:det:inan&gt; [vacacion]{vacaciones}&lt;in:s:gat:noun:inan&gt; muy reljante y [bonito]{bonita}&lt;ga:fm:adj:inan&gt;.</td>\n",
       "      <td>[perfecto]{perfectas}&lt;ga:na:fm:ps:adj:inan&gt;</td>\n",
       "      <td>ga:na:fm:ps:adj:inan</td>\n",
       "      <td>perfectas</td>\n",
       "      <td>perfecto</td>\n",
       "      <td>[las, perfectas, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]</td>\n",
       "      <td>[las, perfecto, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]</td>\n",
       "      <td>[1, 13]</td>\n",
       "      <td>gender and number agreement</td>\n",
       "      <td>[0, 12]</td>\n",
       "      <td>las perfectas vacaciones es una vacaciones muy reljante y bonita.</td>\n",
       "      <td>las perfecto vacaciones es una vacaciones muy reljante y bonita.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1548</td>\n",
       "      <td>35dfbba951e96c672c60bdb33ffd2ad4523ea3a2e55a42404cd1451ac675b447</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>Es el mas azul playa en California y mi favorito playa en el mundo.</td>\n",
       "      <td>Es [el]{la}&lt;ga:fm:det:inan&gt; mas azul playa en California y mi [favorito]{favorita}&lt;ga:fm:adj:inan&gt; playa en el mundo.</td>\n",
       "      <td>[favorito]{favorita}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>ga:fm:adj:inan</td>\n",
       "      <td>favorita</td>\n",
       "      <td>favorito</td>\n",
       "      <td>[Es, la, mas, azul, playa, en, California, y, mi, favorita, playa, en, el, mundo, .]</td>\n",
       "      <td>[Es, la, mas, azul, playa, en, California, y, mi, favorito, playa, en, el, mundo, .]</td>\n",
       "      <td>[9, 25]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[10, 11, 26, 27]</td>\n",
       "      <td>Es la mas azul playa en California y mi favorita playa en el mundo.</td>\n",
       "      <td>Es la mas azul playa en California y mi favorito playa en el mundo.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1138</td>\n",
       "      <td>86117767a230e4bc9a5f64dbbfc5fd6b08bc8d56670d47fb07c9056687f211f6</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>\\n\\nEn la primera día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "      <td>En [la]{el}&lt;ga:mf:det:aty:inan&gt; [primera]{primer}&lt;ga:mf:det:aty:inan&gt; día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "      <td>[la]{el}&lt;ga:mf:det:aty:inan&gt;</td>\n",
       "      <td>ga:mf:det:aty:inan</td>\n",
       "      <td>el</td>\n",
       "      <td>la</td>\n",
       "      <td>[En, el, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]</td>\n",
       "      <td>[En, la, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]</td>\n",
       "      <td>[1, 22]</td>\n",
       "      <td>gender agreement</td>\n",
       "      <td>[2, 23]</td>\n",
       "      <td>En el primer día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "      <td>En la primer día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5212 rows × 16 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0a28d341-023f-4558-9383-533cf01987b2')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-0a28d341-023f-4558-9383-533cf01987b2 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-0a28d341-023f-4558-9383-533cf01987b2');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-2c39d29e-2df9-48b3-83ef-92431d032a1d\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2c39d29e-2df9-48b3-83ef-92431d032a1d')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-2c39d29e-2df9-48b3-83ef-92431d032a1d button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id  \\\n",
       "0             13   \n",
       "1             18   \n",
       "10            71   \n",
       "13            93   \n",
       "32           139   \n",
       "..           ...   \n",
       "43          1137   \n",
       "45          1205   \n",
       "46          1384   \n",
       "47          1548   \n",
       "48          1138   \n",
       "\n",
       "                                                         sentence_id course  \\\n",
       "0   452c9dc1a268e1cd5d57357198dc84aa00deacae5ec6c2007fbd5b22550e740e  SPA 1   \n",
       "1   e627490f6722ef9caeb07e150107b31bcc9c088016a5791570f59dedab58433b  SPA 2   \n",
       "10  32941ba77d49c383a207ba106c4f17ba3cdd3b65d1743530dfc1706b1b3d8be6  SPA 1   \n",
       "13  b415e59e20c87c8daba6816136d38e6dfc1fc005c53019b66aacc878ce1ccdc2  SPA 2   \n",
       "32  7fcc1b964f356d5b3dad9225764cb914dda11c2bea8138b58384f73829ab156f  SPA 2   \n",
       "..                                                               ...    ...   \n",
       "43  4dc889230c51fb53fa2008f196ac32ef1ae3c8588f4e1ec6323e3a56df482e63  SPA 3   \n",
       "45  ddca439dbc58c89c77ee3910845957961a571cdcad0f857007e0486095ae349b  SPA 1   \n",
       "46  c1ca2f5c15fe7682ab4f1503799d0f9f2300b495301aaa2735beefa62f7c64b6  SPA 3   \n",
       "47  35dfbba951e96c672c60bdb33ffd2ad4523ea3a2e55a42404cd1451ac675b447  SPA 2   \n",
       "48  86117767a230e4bc9a5f64dbbfc5fd6b08bc8d56670d47fb07c9056687f211f6  SPA 2   \n",
       "\n",
       "                                                                                                 essay_new  \\\n",
       "0                     No confundas Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1           Un Otro famoso actor, que llama Nicolas Cage, ayudo Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                            Yo miro él en la television.   \n",
       "13                                              Yo quiero conocer Taylor Swift porque ella no es aburrida.   \n",
       "32                     Yo admiro Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "..                                                                                                     ...   \n",
       "43                                                 Para la primera día, espero que comer más comida local.   \n",
       "45                                                  Nosotros traemos la gata en el resto de la vacaciones.   \n",
       "46                                            El perfecto vacaciones es un vacacion muy reljante y bonito.   \n",
       "47                                     Es el mas azul playa en California y mi favorito playa en el mundo.   \n",
       "48  \\n\\nEn la primera día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.   \n",
       "\n",
       "                                                                                                                                                                                                 annotated_sentence  \\\n",
       "0                                                                                                              No confundas []{a}<az:do:an> Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1                                                                                                    Un Otro famoso actor, que llama Nicolas Cage, ayudo []{a}<az:do:an> Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                                                                                                                     Yo miro []{a}<az:do:an> él en la television.   \n",
       "13                                                                                                                                       Yo quiero conocer []{a}<az:do:an> Taylor Swift porque ella no es aburrida.   \n",
       "32                                                                                                              Yo admiro []{a}<az:do:an> Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "..                                                                                                                                                                                                              ...   \n",
       "43                                                                                                  Para [la]{el}<ga:mf:det:aty:inan> [primera]{primer}<ga:mf:det:aty:inan> día, espero que comer más comida local.   \n",
       "45                                                                                                         [Nosotros]{nosotras}<ga:fm:pron:an> traemos la gata en el resto de [la]{las}<na:ps:det:inan> vacaciones.   \n",
       "46  [El]{las}<ga:na:fm:ps:det:inan> [perfecto]{perfectas}<ga:na:fm:ps:adj:inan> vacaciones es [un]{una}<ga:fm:det:inan> [vacacion]{vacaciones}<in:s:gat:noun:inan> muy reljante y [bonito]{bonita}<ga:fm:adj:inan>.   \n",
       "47                                                                                            Es [el]{la}<ga:fm:det:inan> mas azul playa en California y mi [favorito]{favorita}<ga:fm:adj:inan> playa en el mundo.   \n",
       "48                                                       En [la]{el}<ga:mf:det:aty:inan> [primera]{primer}<ga:mf:det:aty:inan> día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.   \n",
       "\n",
       "                                 full_error_tag             error_tag  \\\n",
       "0                               []{a}<az:do:an>              az:do:an   \n",
       "1                               []{a}<az:do:an>              az:do:an   \n",
       "10                              []{a}<az:do:an>              az:do:an   \n",
       "13                              []{a}<az:do:an>              az:do:an   \n",
       "32                              []{a}<az:do:an>              az:do:an   \n",
       "..                                          ...                   ...   \n",
       "43                 [la]{el}<ga:mf:det:aty:inan>    ga:mf:det:aty:inan   \n",
       "45          [Nosotros]{nosotras}<ga:fm:pron:an>         ga:fm:pron:an   \n",
       "46  [perfecto]{perfectas}<ga:na:fm:ps:adj:inan>  ga:na:fm:ps:adj:inan   \n",
       "47         [favorito]{favorita}<ga:fm:adj:inan>        ga:fm:adj:inan   \n",
       "48                 [la]{el}<ga:mf:det:aty:inan>    ga:mf:det:aty:inan   \n",
       "\n",
       "   correction     error  \\\n",
       "0           a       NaN   \n",
       "1           a       NaN   \n",
       "10          a       NaN   \n",
       "13          a       NaN   \n",
       "32          a       NaN   \n",
       "..        ...       ...   \n",
       "43         el        la   \n",
       "45   nosotras  Nosotros   \n",
       "46  perfectas  perfecto   \n",
       "47   favorita  favorito   \n",
       "48         el        la   \n",
       "\n",
       "                                                                                                                      target  \\\n",
       "0                  [No, confundas, a, Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1   [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, a, Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                  [Yo, miro, a, él, en, la, television, .]   \n",
       "13                                                [Yo, quiero, conocer, a, Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                    [Yo, admiro, a, Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "..                                                                                                                       ...   \n",
       "43                                                     [Para, el, primer, día, ,, espero, que, comer, más, comida, local, .]   \n",
       "45                                                      [nosotras, traemos, la, gata, en, el, resto, de, las, vacaciones, .]   \n",
       "46                                            [las, perfectas, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]   \n",
       "47                                      [Es, la, mas, azul, playa, en, California, y, mi, favorita, playa, en, el, mundo, .]   \n",
       "48  [En, el, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]   \n",
       "\n",
       "                                                                                                                           source  \\\n",
       "0                  [No, confundas, [NONE], Selena, Gomez, con, Selena, Quintanilla, ,, de, quien, Selena, Gomez, es, nombrada, .]   \n",
       "1   [Un, Otro, famoso, actor, ,, que, llama, Nicolas, Cage, ,, ayudo, [NONE], Johnny, Depp, empezar, su, profesión, de, actor, .]   \n",
       "10                                                                                  [Yo, miro, [NONE], él, en, la, television, .]   \n",
       "13                                                [Yo, quiero, conocer, [NONE], Taylor, Swift, porque, ella, no, es, aburrida, .]   \n",
       "32                    [Yo, admiro, [NONE], Ian, Somerhalder, porque, yo, estoy, interesado, en, ayudando, el, medio, ambiente, .]   \n",
       "..                                                                                                                            ...   \n",
       "43                                                          [Para, la, primer, día, ,, espero, que, comer, más, comida, local, .]   \n",
       "45                                                           [Nosotros, traemos, la, gata, en, el, resto, de, las, vacaciones, .]   \n",
       "46                                                  [las, perfecto, vacaciones, es, una, vacaciones, muy, reljante, y, bonita, .]   \n",
       "47                                           [Es, la, mas, azul, playa, en, California, y, mi, favorito, playa, en, el, mundo, .]   \n",
       "48       [En, la, primer, día, ,, vamos, al, aeropuerto, de, Los, Ángeles, en, la, noche, del, aeropuerto, de, San, Francisco, .]   \n",
       "\n",
       "   correction_index               new_error_type  \\\n",
       "0           [2, 19]                      article   \n",
       "1          [11, 32]                      article   \n",
       "10          [2, 11]                      article   \n",
       "13          [3, 16]                      article   \n",
       "32          [2, 18]                      article   \n",
       "..              ...                          ...   \n",
       "43          [1, 14]             gender agreement   \n",
       "45          [0, 12]             gender agreement   \n",
       "46          [1, 13]  gender and number agreement   \n",
       "47          [9, 25]             gender agreement   \n",
       "48          [1, 22]             gender agreement   \n",
       "\n",
       "                               final_evident_index  \\\n",
       "0       [1, 3, 4, 5, 6, 7, 18, 20, 21, 22, 23, 24]   \n",
       "1   [7, 8, 10, 12, 13, 14, 28, 29, 31, 33, 34, 35]   \n",
       "10       [0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 14, 15]   \n",
       "13                           [2, 4, 5, 15, 17, 18]   \n",
       "32                           [1, 3, 4, 17, 19, 20]   \n",
       "..                                             ...   \n",
       "43                                         [2, 15]   \n",
       "45                                         [3, 15]   \n",
       "46                                         [0, 12]   \n",
       "47                                [10, 11, 26, 27]   \n",
       "48                                         [2, 23]   \n",
       "\n",
       "                                                                                      target_sentence  \\\n",
       "0              No confundas a Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.   \n",
       "1    Un Otro famoso actor, que llama Nicolas Cage, ayudo a Johnny Depp empezar su profesión de actor.   \n",
       "10                                                                     Yo miro a él en la television.   \n",
       "13                                       Yo quiero conocer a Taylor Swift porque ella no es aburrida.   \n",
       "32              Yo admiro a Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.   \n",
       "..                                                                                                ...   \n",
       "43                                             Para el primer día, espero que comer más comida local.   \n",
       "45                                            nosotras traemos la gata en el resto de las vacaciones.   \n",
       "46                                  las perfectas vacaciones es una vacaciones muy reljante y bonita.   \n",
       "47                                Es la mas azul playa en California y mi favorita playa en el mundo.   \n",
       "48  En el primer día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.   \n",
       "\n",
       "                                                                                          source_sentence  \n",
       "0             No confundas [NONE] Selena Gomez con Selena Quintanilla, de quien Selena Gomez es nombrada.  \n",
       "1   Un Otro famoso actor, que llama Nicolas Cage, ayudo [NONE] Johnny Depp empezar su profesión de actor.  \n",
       "10                                                                    Yo miro [NONE] él en la television.  \n",
       "13                                      Yo quiero conocer [NONE] Taylor Swift porque ella no es aburrida.  \n",
       "32             Yo admiro [NONE] Ian Somerhalder porque yo estoy interesado en ayudando el medio ambiente.  \n",
       "..                                                                                                    ...  \n",
       "43                                                 Para la primer día, espero que comer más comida local.  \n",
       "45                                                Nosotros traemos la gata en el resto de las vacaciones.  \n",
       "46                                       las perfecto vacaciones es una vacaciones muy reljante y bonita.  \n",
       "47                                    Es la mas azul playa en California y mi favorito playa en el mundo.  \n",
       "48      En la primer día, vamos al aeropuerto de Los Ángeles en la noche del aeropuerto de San Francisco.  \n",
       "\n",
       "[5212 rows x 16 columns]"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_evident"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HC6iH4FyO3MK"
   },
   "source": [
    "## Final Sanity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "-W_vjSDfiGla",
    "outputId": "24643ac5-8de2-4cb0-c48d-6810c4bdadcf"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'evidence words = \"el\", \"vacaciones\"\\nindexes = 0, 1'"
      ]
     },
     "execution_count": 397,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_evidence_word('el vacaciones perfectas','el', 'las')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "id": "7s2Pteve7vw0",
    "outputId": "81e8544c-5832-4d5c-c043-4a0e195830a6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-b5433061-4565-48ab-b0d8-33e79b167ccf\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>anno_error_cnt</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>order_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>error_type</th>\n",
       "      <th>source</th>\n",
       "      <th>new_error_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1138</td>\n",
       "      <td>ca39133443908048a335cb11bc4c8aa07b24db065899a7156f99ac61a1c24414</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>El vacación perfecto</td>\n",
       "      <td>[El]{las}&lt;ga:fm:det:inan&gt; [vacación]{vacaciones}&lt;in:s:gat:noun:inan&gt; [perfecto]{perfectas}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>3</td>\n",
       "      <td>[El]{las}&lt;ga:fm:det:inan&gt;</td>\n",
       "      <td>0</td>\n",
       "      <td>ga:fm:det:inan</td>\n",
       "      <td>las</td>\n",
       "      <td>El</td>\n",
       "      <td>[las, vacaciones, perfectas]</td>\n",
       "      <td>[0, 4]</td>\n",
       "      <td>ga:fm</td>\n",
       "      <td>[El, vacaciones, perfectas]</td>\n",
       "      <td>gender agreement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1138</td>\n",
       "      <td>ca39133443908048a335cb11bc4c8aa07b24db065899a7156f99ac61a1c24414</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>El vacación perfecto</td>\n",
       "      <td>[El]{las}&lt;ga:fm:det:inan&gt; [vacación]{vacaciones}&lt;in:s:gat:noun:inan&gt; [perfecto]{perfectas}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>3</td>\n",
       "      <td>[vacación]{vacaciones}&lt;in:s:gat:noun:inan&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>in:s:gat:noun:inan</td>\n",
       "      <td>vacaciones</td>\n",
       "      <td>vacación</td>\n",
       "      <td>[las, vacaciones, perfectas]</td>\n",
       "      <td>[1, 5]</td>\n",
       "      <td>in:s:gat:noun</td>\n",
       "      <td>[las, vacación, perfectas]</td>\n",
       "      <td>number agreement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1138</td>\n",
       "      <td>ca39133443908048a335cb11bc4c8aa07b24db065899a7156f99ac61a1c24414</td>\n",
       "      <td>SPA 2</td>\n",
       "      <td>El vacación perfecto</td>\n",
       "      <td>[El]{las}&lt;ga:fm:det:inan&gt; [vacación]{vacaciones}&lt;in:s:gat:noun:inan&gt; [perfecto]{perfectas}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>3</td>\n",
       "      <td>[perfecto]{perfectas}&lt;ga:fm:adj:inan&gt;</td>\n",
       "      <td>2</td>\n",
       "      <td>ga:fm:adj:inan</td>\n",
       "      <td>perfectas</td>\n",
       "      <td>perfecto</td>\n",
       "      <td>[las, vacaciones, perfectas]</td>\n",
       "      <td>[2, 6]</td>\n",
       "      <td>ga:fm</td>\n",
       "      <td>[las, vacaciones, perfecto]</td>\n",
       "      <td>gender agreement</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b5433061-4565-48ab-b0d8-33e79b167ccf')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-b5433061-4565-48ab-b0d8-33e79b167ccf button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-b5433061-4565-48ab-b0d8-33e79b167ccf');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-8929fd11-2326-45b9-8799-f71699e56957\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-8929fd11-2326-45b9-8799-f71699e56957')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-8929fd11-2326-45b9-8799-f71699e56957 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "    paragraph_id  \\\n",
       "37          1138   \n",
       "37          1138   \n",
       "37          1138   \n",
       "\n",
       "                                                         sentence_id course  \\\n",
       "37  ca39133443908048a335cb11bc4c8aa07b24db065899a7156f99ac61a1c24414  SPA 2   \n",
       "37  ca39133443908048a335cb11bc4c8aa07b24db065899a7156f99ac61a1c24414  SPA 2   \n",
       "37  ca39133443908048a335cb11bc4c8aa07b24db065899a7156f99ac61a1c24414  SPA 2   \n",
       "\n",
       "               essay_new  \\\n",
       "37  El vacación perfecto   \n",
       "37  El vacación perfecto   \n",
       "37  El vacación perfecto   \n",
       "\n",
       "                                                                                            annotated_sentence  \\\n",
       "37  [El]{las}<ga:fm:det:inan> [vacación]{vacaciones}<in:s:gat:noun:inan> [perfecto]{perfectas}<ga:fm:adj:inan>   \n",
       "37  [El]{las}<ga:fm:det:inan> [vacación]{vacaciones}<in:s:gat:noun:inan> [perfecto]{perfectas}<ga:fm:adj:inan>   \n",
       "37  [El]{las}<ga:fm:det:inan> [vacación]{vacaciones}<in:s:gat:noun:inan> [perfecto]{perfectas}<ga:fm:adj:inan>   \n",
       "\n",
       "    anno_error_cnt                              full_error_tag order_tag  \\\n",
       "37               3                   [El]{las}<ga:fm:det:inan>         0   \n",
       "37               3  [vacación]{vacaciones}<in:s:gat:noun:inan>         1   \n",
       "37               3       [perfecto]{perfectas}<ga:fm:adj:inan>         2   \n",
       "\n",
       "             error_tag  correction     error                        target  \\\n",
       "37      ga:fm:det:inan         las        El  [las, vacaciones, perfectas]   \n",
       "37  in:s:gat:noun:inan  vacaciones  vacación  [las, vacaciones, perfectas]   \n",
       "37      ga:fm:adj:inan   perfectas  perfecto  [las, vacaciones, perfectas]   \n",
       "\n",
       "   correction_index     error_type                       source  \\\n",
       "37           [0, 4]          ga:fm  [El, vacaciones, perfectas]   \n",
       "37           [1, 5]  in:s:gat:noun   [las, vacación, perfectas]   \n",
       "37           [2, 6]          ga:fm  [las, vacaciones, perfecto]   \n",
       "\n",
       "      new_error_type  \n",
       "37  gender agreement  \n",
       "37  number agreement  \n",
       "37  gender agreement  "
      ]
     },
     "execution_count": 383,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_combine[df_combine['sentence_id']=='ca39133443908048a335cb11bc4c8aa07b24db065899a7156f99ac61a1c24414']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zfEy2u5ZkADt",
    "outputId": "6cfdc20e-7dec-4236-e908-fc032fed694f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5223, 16)"
      ]
     },
     "execution_count": 401,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_combine.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1BSQU_aKO3ML",
    "outputId": "4e27d7ea-56ac-4b52-e96c-411060e7bee2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "paragraph_id          5223\n",
       "sentence_id           5223\n",
       "course                5223\n",
       "essay_new             5223\n",
       "annotated_sentence    5223\n",
       "anno_error_cnt        5223\n",
       "full_error_tag        5223\n",
       "order_tag             5223\n",
       "error_tag             5223\n",
       "correction            5223\n",
       "error                 5223\n",
       "target                5223\n",
       "correction_index      5223\n",
       "error_type            5223\n",
       "source                5223\n",
       "check_target          5223\n",
       "check_source          5223\n",
       "dtype: int64"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_combine[df_combine.notnull()].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HXwMyMBxi70e",
    "outputId": "dcceb30d-6381-4f60-da8f-440976c8436a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], dtype=object)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_combine[df_combine['error_type'].isnull()].error_tag.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YbbpWp6c1PsY"
   },
   "outputs": [],
   "source": [
    "pd.options.display.max_colwidth = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zi9jx2dT6S7R"
   },
   "outputs": [],
   "source": [
    "def check(list_item):\n",
    "  sum = 0\n",
    "  pattern  = r'\\}'\n",
    "  for i in list_item:\n",
    "    if len(re.findall(pattern, i)) > 0:\n",
    "      sum += 1\n",
    "  return sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uhluuA8Q1ckp"
   },
   "outputs": [],
   "source": [
    "df_combine['check_target'] = df_combine['target'].apply(check)\n",
    "df_combine['check_source'] = df_combine['source'].apply(check)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_avh7RhG5pbt"
   },
   "outputs": [],
   "source": [
    "target = df_combine[df_combine['check_target'] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 73
    },
    "id": "DX7_LomBPZGg",
    "outputId": "407ad402-e7bf-461f-d8b8-c325dc990cd3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-85e52dce-dfe4-4aed-b8a7-7c2a7b057dbc\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>course</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>annotated_sentence</th>\n",
       "      <th>anno_error_cnt</th>\n",
       "      <th>full_error_tag</th>\n",
       "      <th>order_tag</th>\n",
       "      <th>error_tag</th>\n",
       "      <th>correction</th>\n",
       "      <th>error</th>\n",
       "      <th>target</th>\n",
       "      <th>correction_index</th>\n",
       "      <th>error_type</th>\n",
       "      <th>source</th>\n",
       "      <th>check_target</th>\n",
       "      <th>check_source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-85e52dce-dfe4-4aed-b8a7-7c2a7b057dbc')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-85e52dce-dfe4-4aed-b8a7-7c2a7b057dbc button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-85e52dce-dfe4-4aed-b8a7-7c2a7b057dbc');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [paragraph_id, sentence_id, course, essay_new, annotated_sentence, anno_error_cnt, full_error_tag, order_tag, error_tag, correction, error, target, correction_index, error_type, source, check_target, check_source]\n",
       "Index: []"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vHXCsLFDSiaJ"
   },
   "outputs": [],
   "source": [
    "target.to_csv('check.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h-cY__XHMwNE",
    "outputId": "b360d100-94b4-450b-c409-ad7c5122cf73"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 17)"
      ]
     },
     "execution_count": 314,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xbSxb2od7Zsl"
   },
   "outputs": [],
   "source": [
    "source = df_combine[df_combine['check_source'] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pbj9kmtCMzqt",
    "outputId": "e6da9df2-e098-426c-830e-99d17050fbb0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 17)"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xzJ7y24D64ZQ"
   },
   "outputs": [],
   "source": [
    "df_combine.to_csv('final_dataset.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C8YBLSuoO3ML"
   },
   "source": [
    "## Saving the df to google drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cyBRlGV6O3ML",
    "outputId": "3716125a-909d-4adf-930e-5b6c0184f344"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k_j0Dp3ZO3ML"
   },
   "outputs": [],
   "source": [
    "well_formated_df.to_csv('well_formated_df.csv')\n",
    "bad_formated_df.to_csv('bad_formated_df.csv')\n",
    "!cp well_formated_df.csv \"/content/gdrive/MyDrive/\"\n",
    "!cp bad_formated_df.csv \"/content/gdrive/MyDrive/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "azezS6BMO3ML",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Old code needs checking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PqVmkFfsqTxv",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### check if all input has the same sentences lengths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LQctBt067xq_"
   },
   "source": [
    "Compare which way of splitting works better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fjDITVTPXL3q"
   },
   "outputs": [],
   "source": [
    "# #split into list\n",
    "# df['essay_new'] = df['essay'].apply(split_paragraph_with_nltk)\n",
    "# df['p_annotator1'] = df['a personal annotator1'].apply(split_paragraph_with_nltk)\n",
    "# df['p_annotator2'] = df['a personal annotator2'].apply(split_paragraph_with_nltk)\n",
    "# df['g_annotator1'] = df['gender-number annotator1'].apply(split_paragraph_with_nltk)\n",
    "# df['g_annotator2'] = df['gender-number annotator2'].apply(split_paragraph_with_nltk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WCz5dKZ_bFMu"
   },
   "outputs": [],
   "source": [
    "# df['p_annotator1_cnt'] =  df['p_annotator1'].apply(lambda x: len(x))\n",
    "# df['p_annotator2_cnt'] =  df['p_annotator2'].apply(lambda x: len(x))\n",
    "# df['g_annotator1_cnt'] =  df['g_annotator1'].apply(lambda x: len(x))\n",
    "# df['g_annotator2_cnt'] =  df['g_annotator2'].apply(lambda x: len(x))\n",
    "# df['essay_new_cnt'] =  df['essay_new'].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YR7DS31KbJLF"
   },
   "outputs": [],
   "source": [
    "# #not all the paragraphs have the same length\n",
    "# # length with nltk\n",
    "# print(df[df['p_annotator1_cnt'] != df['p_annotator2_cnt']].shape)\n",
    "# print(df[df['essay_new_cnt'] != df['p_annotator1_cnt']].shape)\n",
    "# print(df[df['essay_new_cnt'] != df['g_annotator1_cnt']].shape)  # the g_annotator is empty\n",
    "# print(df[df['g_annotator1_cnt'] != df['g_annotator2_cnt']].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2zwKgoMxO3MM"
   },
   "outputs": [],
   "source": [
    "# df_final = df_final.replace('', np.nan)\n",
    "# df_final = df_final.replace(' ', np.nan)\n",
    "# df_final = df_final.replace('\\n', np.nan)\n",
    "# df_final = df_final.replace('\"\\n', np.nan)\n",
    "# df_final = df_final.replace('\"\\n\\n', np.nan)\n",
    "# df_final = df_final.replace('\\n\\n', np.nan)\n",
    "# df_final = df_final.replace('\"\\n\\n\\n', np.nan)\n",
    "\n",
    "# df_final = df_final.replace('\"\\n__\\n', np.nan)\n",
    "# df_final = df_final.replace('\"', np.nan)\n",
    "\n",
    "# # TODO: There are still some rows with jsut white spaces in essay column that needs to be nuked.\n",
    "# # TODO: Need to remove rows with very few letters/words."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PZigvPEyO3MM",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Review Source vs G Annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZVuVV6JtO3MM"
   },
   "outputs": [],
   "source": [
    "# diff(df_final, 'essay_new', 'g_annotator_list', n=20, dropna=False, colwidth=65, onlydiffs=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PJinY_58O3MM",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Review Source vs P annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-ADRnQ-hO3MN"
   },
   "outputs": [],
   "source": [
    "# diff(df_final, 'essay_new', 'p_annotator_list', n=3, dropna=False, colwidth=65, onlydiffs=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "J5YKt7azqn9V",
    "tags": []
   },
   "source": [
    "## Analyzing encoded error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4ulfiB18jPiV"
   },
   "outputs": [],
   "source": [
    "pattern = r'\\[.*?\\]\\{.*?\\}\\<.*?\\>'\n",
    "\n",
    "df_final['p_anno_error_cnt'] = df_final['p_annotator_list'].str.count(pattern)\n",
    "df_final['g_anno_error_cnt'] = df_final['g_annotator_list'].str.count(pattern)\n",
    "df_final['p_anno_error_cnt'] = df_final['p_anno_error_cnt'].fillna(0)\n",
    "df_final['g_anno_error_cnt'] = df_final['g_anno_error_cnt'].fillna(0)\n",
    "df_final = df_final.astype({\"p_anno_error_cnt\":'int', \"g_anno_error_cnt\":'int'})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "vyIyAq-SjpRY",
    "outputId": "6fc1e93f-13bf-4cf8-c35b-288393ff8734"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>is_duplicated</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>p_annotator_list</th>\n",
       "      <th>g_annotator_list</th>\n",
       "      <th>p_anno_error_cnt</th>\n",
       "      <th>g_anno_error_cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Una persona famosa que admiro es Lauren Jauregui</td>\n",
       "      <td>Una persona famosa que admiro es Lauren Jauregui</td>\n",
       "      <td>Una persona famosa que admiro es Lauren Jauregui</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Ella es una cantante en el grupo Fifth Harmony</td>\n",
       "      <td>Ella es una cantante en el grupo Fifth Harmony</td>\n",
       "      <td>Ella es una cantante en el grupo Fifth Harmony</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Ella ha asumido su papel de cantante que much...</td>\n",
       "      <td>Ella ha asumido su papel de cantante que much...</td>\n",
       "      <td>Ella ha asumido su papel de cantante que much...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Ella habla en contra del fanatismo y el odio ...</td>\n",
       "      <td>Ella habla en contra del fanatismo y el odio ...</td>\n",
       "      <td>Ella habla en contra del fanatismo y el odio ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Muchas niñas, especialmente las niñas que son...</td>\n",
       "      <td>Muchas niñas, especialmente las niñas que son...</td>\n",
       "      <td>Muchas niñas, especialmente las niñas que son...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>Entonces, Hawaii es la casa a muchos frutas t...</td>\n",
       "      <td>Entonces, Hawaii es la casa a muchos frutas t...</td>\n",
       "      <td>Entonces, Hawaii es la casa a [muchos]{muchas...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>Hawaii es un especial isla y tiene algunos vo...</td>\n",
       "      <td>Hawaii es un especial isla y tiene algunos vo...</td>\n",
       "      <td>Hawaii es [un]{una}&lt;ga:fm:det:inan&gt; especial ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>EN la isla largo, tu veas (lava) ir en la oce...</td>\n",
       "      <td>EN la isla largo, tu veas (lava) ir en la oce...</td>\n",
       "      <td>EN la isla [largo]{larga}&lt;ga:fm:adj:inan&gt;, tu...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>\\n\\nYo quiero ir a Hwaii con mi novio prque e...</td>\n",
       "      <td>\\n\\nYo quiero ir a Hwaii con mi novio prque e...</td>\n",
       "      <td>\\n\\nYo quiero ir a Hwaii con mi novio prque e...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>Yo soy muy emocionada que yo voy a Hawaii est...</td>\n",
       "      <td>Yo soy muy emocionada que yo voy a Hawaii est...</td>\n",
       "      <td>Yo soy muy emocionada que yo voy a Hawaii est...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22941 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  paragraph_id  is_duplicated  \\\n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "..      ...           ...            ...   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "\n",
       "                                             essay_new  \\\n",
       "0     Una persona famosa que admiro es Lauren Jauregui   \n",
       "0       Ella es una cantante en el grupo Fifth Harmony   \n",
       "0     Ella ha asumido su papel de cantante que much...   \n",
       "0     Ella habla en contra del fanatismo y el odio ...   \n",
       "0     Muchas niñas, especialmente las niñas que son...   \n",
       "..                                                 ...   \n",
       "167   Entonces, Hawaii es la casa a muchos frutas t...   \n",
       "167   Hawaii es un especial isla y tiene algunos vo...   \n",
       "167   EN la isla largo, tu veas (lava) ir en la oce...   \n",
       "167   \\n\\nYo quiero ir a Hwaii con mi novio prque e...   \n",
       "167   Yo soy muy emocionada que yo voy a Hawaii est...   \n",
       "\n",
       "                                      p_annotator_list  \\\n",
       "0     Una persona famosa que admiro es Lauren Jauregui   \n",
       "0       Ella es una cantante en el grupo Fifth Harmony   \n",
       "0     Ella ha asumido su papel de cantante que much...   \n",
       "0     Ella habla en contra del fanatismo y el odio ...   \n",
       "0     Muchas niñas, especialmente las niñas que son...   \n",
       "..                                                 ...   \n",
       "167   Entonces, Hawaii es la casa a muchos frutas t...   \n",
       "167   Hawaii es un especial isla y tiene algunos vo...   \n",
       "167   EN la isla largo, tu veas (lava) ir en la oce...   \n",
       "167   \\n\\nYo quiero ir a Hwaii con mi novio prque e...   \n",
       "167   Yo soy muy emocionada que yo voy a Hawaii est...   \n",
       "\n",
       "                                      g_annotator_list  p_anno_error_cnt  \\\n",
       "0     Una persona famosa que admiro es Lauren Jauregui                 0   \n",
       "0       Ella es una cantante en el grupo Fifth Harmony                 0   \n",
       "0     Ella ha asumido su papel de cantante que much...                 0   \n",
       "0     Ella habla en contra del fanatismo y el odio ...                 0   \n",
       "0     Muchas niñas, especialmente las niñas que son...                 0   \n",
       "..                                                 ...               ...   \n",
       "167   Entonces, Hawaii es la casa a [muchos]{muchas...                 0   \n",
       "167   Hawaii es [un]{una}<ga:fm:det:inan> especial ...                 0   \n",
       "167   EN la isla [largo]{larga}<ga:fm:adj:inan>, tu...                 0   \n",
       "167   \\n\\nYo quiero ir a Hwaii con mi novio prque e...                 0   \n",
       "167   Yo soy muy emocionada que yo voy a Hawaii est...                 0   \n",
       "\n",
       "     g_anno_error_cnt  \n",
       "0                   0  \n",
       "0                   0  \n",
       "0                   0  \n",
       "0                   0  \n",
       "0                   3  \n",
       "..                ...  \n",
       "167                 2  \n",
       "167                 1  \n",
       "167                 1  \n",
       "167                 0  \n",
       "167                 1  \n",
       "\n",
       "[22941 rows x 8 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XK-w0rgqt5se"
   },
   "source": [
    "### Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ByfkcGKyuao5"
   },
   "source": [
    "% of sentences with grammartical mistake."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "id": "txBLzpNyqmTh",
    "outputId": "001f5d04-d97f-4111-90b7-1420ec5df7f3"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAGwCAYAAAC0HlECAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABgpUlEQVR4nO3de1xUdf4/8NcZLsN9AJGbAt5NBRW0FC0zM9S85NY3LV3StWjbvCZuZfvb1W5itbVtWZu1ZjeLdtdLZYaSF8wEL1zyhncUVBDFmeEiDDDz+f2BHB0BndGBMwOv5+MxDznnfM6Z9xnJefU5n/M5khBCgIiIiIhuSKV0AURERESOgKGJiIiIyAIMTUREREQWYGgiIiIisgBDExEREZEFGJqIiIiILMDQRERERGQBZ6ULaE1MJhPOnTsHb29vSJKkdDlERERkASEEysrKEBoaCpWq6f4khiYbOnfuHMLCwpQug4iIiG5BQUEBOnbs2OR2hiYb8vb2BlD3ofv4+ChcDREREVmitLQUYWFh8vd4UxiabKj+kpyPjw9DExERkYO52dAaDgQnIiIisoCioSkpKQl33nknvL29ERgYiIkTJ+LIkSNmbYQQWLx4MUJDQ+Hu7o7hw4fj4MGDZm0MBgNmz56NgIAAeHp6YsKECThz5oxZG61Wi/j4eGg0Gmg0GsTHx0On05m1yc/Px/jx4+Hp6YmAgADMmTMH1dXVzXLuRERE5FgUDU1paWmYOXMmMjIykJqaitraWsTFxaGiokJu8+abb+Kdd97BsmXLsGfPHgQHB+OBBx5AWVmZ3GbevHlYu3YtkpOTsWPHDpSXl2PcuHEwGo1ymylTpiAnJwcpKSlISUlBTk4O4uPj5e1GoxFjx45FRUUFduzYgeTkZKxevRqJiYkt82EQERGRfRN2pLi4WAAQaWlpQgghTCaTCA4OFkuXLpXbVFVVCY1GIz766CMhhBA6nU64uLiI5ORkuc3Zs2eFSqUSKSkpQgghDh06JACIjIwMuU16eroAIA4fPiyEEGLDhg1CpVKJs2fPym2++eYboVarhV6vt6h+vV4vAFjcnoiIiJRn6fe3XY1p0uv1AAB/f38AQF5eHoqKihAXFye3UavVuPfee7Fz504AQGZmJmpqaszahIaGIjIyUm6Tnp4OjUaDQYMGyW0GDx4MjUZj1iYyMhKhoaFym1GjRsFgMCAzM7PReg0GA0pLS81eRERE1DrZTWgSQmD+/Pm4++67ERkZCQAoKioCAAQFBZm1DQoKkrcVFRXB1dUVfn5+N2wTGBjY4D0DAwPN2lz/Pn5+fnB1dZXbXC8pKUkeI6XRaDhHExERUStmN6Fp1qxZ2LdvH7755psG266/BVAIcdPbAq9v01j7W2lzrYULF0Kv18uvgoKCG9ZEREREjssuQtPs2bPx/fffY+vWrWYzcQYHBwNAg56e4uJiuVcoODgY1dXV0Gq1N2xz/vz5Bu974cIFszbXv49Wq0VNTU2DHqh6arVanpOJczMRERG1boqGJiEEZs2ahTVr1mDLli3o3Lmz2fbOnTsjODgYqamp8rrq6mqkpaVhyJAhAIABAwbAxcXFrE1hYSEOHDggt4mNjYVer8fu3bvlNrt27YJerzdrc+DAARQWFsptNm3aBLVajQEDBtj+5ImIiMihSEIIodSbP/vss/j666/x3XffoWfPnvJ6jUYDd3d3AMAbb7yBpKQkrFy5Et27d8eSJUuwbds2HDlyRJ7u/E9/+hPWr1+Pzz77DP7+/liwYAFKSkqQmZkJJycnAMCYMWNw7tw5LF++HADw9NNPIyIiAj/88AOAuikH+vfvj6CgILz11lu4dOkSpk+fjokTJ+L999+36HxKS0uh0Wig1+vZ60REROQgLP7+bua7+G4IQKOvlStXym1MJpNYtGiRCA4OFmq1WgwbNkzs37/f7DiVlZVi1qxZwt/fX7i7u4tx48aJ/Px8szYlJSVi6tSpwtvbW3h7e4upU6cKrVZr1ub06dNi7Nixwt3dXfj7+4tZs2aJqqoqi8+HUw4QERE5Hku/vxXtaWpt2NNERETkeCz9/raLgeB0Y4ZaI7LztTCamG+JiIiUwtBk50wmgcFLNuN3H+7E8eJypcshIiJqsxia7JxKJeGO4Lquwux87U1aExERUXNhaHIA0eG+AIAshiYiIiLFMDQ5gJjwukfEZOfrlC2EiIioDWNocgD9r/Q0HSsuh76yRtliiIiI2iiGJgcQ4KVGuL8HAOC3Ap2yxRAREbVRDE0OIobjmoiIiBTF0OQgojmuiYiISFEMTQ7i6mBwLUyc5JKIiKjFMTQ5iDtCvOHmokJpVS1OXqxQuhwiIqI2h6HJQbg4qdC3gy8ATnJJRESkBIYmB3J1kkudonUQERG1RQxNDiT6mnFNRERE1LIYmhxIfU/T0fNlKDfUKlsMERFRG8PQ5ECCfNzQwdcdJgHs4ySXRERELYqhycHw4b1ERETKYGhyMJzkkoiISBkMTQ6m/nEq2QU6CMFJLomIiFoKQ5OD6R3qA1cnFS5VVON0yWWlyyEiImozGJocjNrZCZEdfAAA2QUc10RERNRSGJocUP24pqzTOmULISIiakMYmhxQtDyuiT1NRERELYWhyQHFXOlpyi0sw+VqTnJJRETUEhiaHFCIxg1BPmoYTQL7z+iVLoeIiKhNYGhyQJIkyb1NfHgvERFRy2BoclDyuCbODE5ERNQiGJocVH1PEye5JCIiahkMTQ4qsoMGzioJF8oMOKOtVLocIiKiVo+hyUG5uTihT2j9JJc6ZYshIiJqAxiaHNjVSS45romIiKi5MTQ5sOhrHt5LREREzUvR0LR9+3aMHz8eoaGhkCQJ69atM9suSVKjr7feektuM3z48AbbH3vsMbPjaLVaxMfHQ6PRQKPRID4+HjqdzqxNfn4+xo8fD09PTwQEBGDOnDmorq5urlO3ifrB4IfO6VFVY1S4GiIiotZN0dBUUVGBfv36YdmyZY1uLywsNHt9+umnkCQJjzzyiFm7hIQEs3bLly832z5lyhTk5OQgJSUFKSkpyMnJQXx8vLzdaDRi7NixqKiowI4dO5CcnIzVq1cjMTHR9idtQx393BHg5Yoao8DBc5zkkoiIqDk5K/nmY8aMwZgxY5rcHhwcbLb83Xff4b777kOXLl3M1nt4eDRoWy83NxcpKSnIyMjAoEGDAACffPIJYmNjceTIEfTs2RObNm3CoUOHUFBQgNDQUADA22+/jenTp+P111+Hj4/P7Zxms5EkCdHhfkg9dB7Z+ToMiPBXuiQiIqJWy2HGNJ0/fx4//vgjnnzyyQbbVq1ahYCAAPTp0wcLFixAWVmZvC09PR0ajUYOTAAwePBgaDQa7Ny5U24TGRkpByYAGDVqFAwGAzIzM5usyWAwoLS01OzV0urHNWVxkksiIqJmpWhPkzU+//xzeHt74+GHHzZbP3XqVHTu3BnBwcE4cOAAFi5ciN9++w2pqakAgKKiIgQGBjY4XmBgIIqKiuQ2QUFBZtv9/Pzg6uoqt2lMUlISXn755ds9tdsiT3LJx6kQERE1K4cJTZ9++immTp0KNzc3s/UJCQnyz5GRkejevTsGDhyIrKwsxMTEAKi7jHU9IYTZekvaXG/hwoWYP3++vFxaWoqwsDDLT8oG+nbUwEkloVBfhUJ9JUI07i36/kRERG2FQ1ye++WXX3DkyBE89dRTN20bExMDFxcXHDt2DEDduKjz5883aHfhwgW5dyk4OLhBj5JWq0VNTU2DHqhrqdVq+Pj4mL1amoerM+4I9gbA3iYiIqLm5BChacWKFRgwYAD69et307YHDx5ETU0NQkJCAACxsbHQ6/XYvXu33GbXrl3Q6/UYMmSI3ObAgQMoLCyU22zatAlqtRoDBgyw8dnYnjyuiZNcEhERNRtFQ1N5eTlycnKQk5MDAMjLy0NOTg7y8/PlNqWlpfjvf//baC/TiRMn8Morr2Dv3r04deoUNmzYgEcffRTR0dEYOnQoAKBXr14YPXo0EhISkJGRgYyMDCQkJGDcuHHo2bMnACAuLg69e/dGfHw8srOzsXnzZixYsAAJCQl2e+fctaLDrj68l4iIiJqHoqFp7969iI6ORnR0NABg/vz5iI6Oxt/+9je5TXJyMoQQePzxxxvs7+rqis2bN2PUqFHo2bMn5syZg7i4OPz8889wcnKS261atQpRUVGIi4tDXFwc+vbtiy+//FLe7uTkhB9//BFubm4YOnQoJk2ahIkTJ+Lvf/97M5697cRE1IWm/Wf1qK41KVwNERFR6yQJIYTSRbQWpaWl0Gg00Ov1LdpDJYRAzKup0F6uwbqZQ9E/zLfF3puIiMjRWfr97RBjmujG6ie5BIBsztdERETULBiaWonoK71LWbyDjoiIqFkwNLUS9eOa2NNERETUPBiaWom+HTWQJOCMthLFZVVKl0NERNTqMDS1Et5uLugRyEkuiYiImgtDUysSE+ELgA/vJSIiag4MTa2IPMkle5qIiIhsjqGpFanvadp3RocaIye5JCIisiWGplakS4AXfNycUVVjwpGiMqXLISIialUYmloRlUpCf05ySURE1CwYmloZTnJJRETUPBiaWhlOcklERNQ8GJpamf4dfQEAp0ouo6TcoGwxRERErQhDUyuj8XBB1/aeAICcAp2yxRAREbUiDE2tUMyVweCc5JKIiMh2GJpaoehwTnJJRERkawxNrVD9JJe/FehgNAlliyEiImolGJpaoe6B3vBSO6Oi2oij5znJJRERkS0wNLVCTioJ/cI0AHiJjoiIyFYYmlqp+of3cjA4ERGRbTA0tVL145o4ySUREZFtMDS1Uv2v9DSduFAB3eVqhashIiJyfAxNrZS/pys6tfMAwEkuiYiIbIGhqRW7OsmlTtlCiIiIWgGGplYsOtwXAMc1ERER2QJDUytWPzN4ToEOJk5ySUREdFsYmlqxO4K94e7ihLKqWpy4UK50OURERA6NoakVc3ZSoW9HTnJJRERkCwxNrVx0OCe5JCIisgWGplbu6mBwnaJ1EBEROTqGplauPjQdLS5DaVWNssUQERE5MIamVi7Q2w0d/dwhBLCvQK90OURERA7L6tCUlZWF/fv3y8vfffcdJk6ciJdeegnV1dY9rmP79u0YP348QkNDIUkS1q1bZ7Z9+vTpkCTJ7DV48GCzNgaDAbNnz0ZAQAA8PT0xYcIEnDlzxqyNVqtFfHw8NBoNNBoN4uPjodPpzNrk5+dj/Pjx8PT0REBAAObMmWP1+dir+kkuOV8TERHRrbM6NP3xj3/E0aNHAQAnT57EY489Bg8PD/z3v//F888/b9WxKioq0K9fPyxbtqzJNqNHj0ZhYaH82rBhg9n2efPmYe3atUhOTsaOHTtQXl6OcePGwWg0ym2mTJmCnJwcpKSkICUlBTk5OYiPj5e3G41GjB07FhUVFdixYweSk5OxevVqJCYmWnU+9qr+Eh0HgxMREd0GYSUfHx9x/PhxIYQQS5cuFXFxcUIIIXbs2CE6duxo7eFkAMTatWvN1k2bNk089NBDTe6j0+mEi4uLSE5OltedPXtWqFQqkZKSIoQQ4tChQwKAyMjIkNukp6cLAOLw4cNCCCE2bNggVCqVOHv2rNzmm2++EWq1Wuj1eovPQa/XCwBW7dMScvK1IuKF9aLfyxuFyWRSuhwiIiK7Yun3t9U9TUIImEwmAMDPP/+MBx98EAAQFhaGixcv2izM1du2bRsCAwPRo0cPJCQkoLi4WN6WmZmJmpoaxMXFyetCQ0MRGRmJnTt3AgDS09Oh0WgwaNAguc3gwYOh0WjM2kRGRiI0NFRuM2rUKBgMBmRmZjZZm8FgQGlpqdnLHvUK8YHaWQXd5RrkXaxQuhwiIiKHZHVoGjhwIF577TV8+eWXSEtLw9ixYwEAeXl5CAoKsmlxY8aMwapVq7Blyxa8/fbb2LNnD0aMGAGDwQAAKCoqgqurK/z8/Mz2CwoKQlFRkdwmMDCwwbEDAwPN2lxfu5+fH1xdXeU2jUlKSpLHSWk0GoSFhd3W+TYXV2cVojpwkksiIqLbYXVoevfdd5GVlYVZs2bhL3/5C7p16wYA+N///ochQ4bYtLjJkydj7NixiIyMxPjx4/HTTz/h6NGj+PHHH2+4nxACkiTJy9f+fDttrrdw4ULo9Xr5VVBQYMlpKYLjmoiIiG6PszWNjUYjtFot0tLS4O/vb7btrbfegpOTk02Lu15ISAgiIiJw7NgxAEBwcDCqq6uh1WrNepuKi4vlABccHIzz5883ONaFCxfk3qXg4GDs2rXLbLtWq0VNTc0Ne8/UajXUavVtn1dLqJsZPI89TURERLfIqp4mJycnjBo1Cnp9w/l+3Nzc4OLiYrPCGlNSUoKCggKEhIQAAAYMGAAXFxekpqbKbQoLC3HgwAE5NMXGxkKv12P37t1ym127dkGv15u1OXDgAAoLC+U2mzZtglqtxoABA5r1nFpK/bQDh4tKUWGoVbgaIiIix2P15bmoqCicPHnSJm9eXl6OnJwc5OTkAKgbF5WTk4P8/HyUl5djwYIFSE9Px6lTp7Bt2zaMHz8eAQEB+N3vfgcA0Gg0ePLJJ5GYmIjNmzcjOzsbv//97xEVFYWRI0cCAHr16oXRo0cjISEBGRkZyMjIQEJCAsaNG4eePXsCAOLi4tC7d2/Ex8cjOzsbmzdvxoIFC5CQkAAfHx+bnKvSgjVuCNG4wSSAfWc4ySUREZHVrL0tb+PGjaJ///7ihx9+EOfOnRN6vd7sZY2tW7cKAA1e06ZNE5cvXxZxcXGiffv2wsXFRYSHh4tp06aJ/Px8s2NUVlaKWbNmCX9/f+Hu7i7GjRvXoE1JSYmYOnWq8Pb2Ft7e3mLq1KlCq9WatTl9+rQYO3ascHd3F/7+/mLWrFmiqqrKqvOx1ykH6j37VaaIeGG9+GDrMaVLISIishuWfn9LQghhTchSqa52Tl07SFpcGTR97aSSbU1paSk0Gg30er1d9lD9+5eTeO3HXIzsFYR/TxuodDlERER2wdLvb6sGggPA1q1bb6swUk70lXFNOQXam94ZSEREROasDk333ntvc9RBLaBPqA9cnCRcLK9GwaVKhLfzULokIiIih2F1aAIAnU6HFStWIDc3F5IkoXfv3pgxYwY0Go2t6yMbcnNxQp9QDXIKdMgu0DI0ERERWcHqu+f27t2Lrl274h//+AcuXbqEixcv4p133kHXrl2RlZXVHDWSDcmTXJ7mJJdERETWsDo0Pffcc5gwYQJOnTqFNWvWYO3atcjLy8O4ceMwb968ZiiRbKl+XFN2gU7ZQoiIiByM1Zfn9u7di08++QTOzld3dXZ2xvPPP4+BA3lHlr2LudLTdOhcKapqjHBzad5Z3ImIiFoLq3uafHx8kJ+f32B9QUEBvL29bVIUNZ8Ovu5o761GrUlg/1lOcklERGQpq0PT5MmT8eSTT+Lbb79FQUEBzpw5g+TkZDz11FN4/PHHm6NGsiFJkuTepmw+vJeIiMhiVl+e+/vf/w5JkvDEE0+gtrbuGWYuLi7405/+hKVLl9q8QLK96HA/bDx4HlmndUqXQkRE5DCsCk1GoxHp6elYtGgRkpKScOLECQgh0K1bN3h48PZ1R1H/8N6sfE5ySUREZCmrQpOTkxNGjRqF3Nxc+Pv7IyoqqrnqomYU1UEDJ5WE4jIDzumr0MHXXemSiIiI7J7VY5qioqJw8uTJ5qiFWoi7qxN6hdQN2ue4JiIiIstYHZpef/11LFiwAOvXr0dhYSFKS0vNXuQY5Et0HNdERERkEasHgo8ePRoAMGHCBLOxMPVjY4xGo+2qo2YTHe6LL9JPI7uAPU1ERESWsDo0bd26tTnqoBZW39N08GwpDLVGqJ05ySUREdGNWBWaampqsHjxYixfvhw9evRorpqoBYT7e8Df0xWXKqpx8FypHKKIiIiocVaNaXJxccGBAwd4i3orYD7JpU7RWoiIiByB1QPBn3jiCaxYsaI5aqEWFn3NfE1ERER0Y1aPaaqursa///1vpKamYuDAgfD09DTb/s4779isOGpe0Vd6mnLY00RERHRTVoemAwcOICYmBgBw9OhRs228bOdY+nb0hUoCzuoqcb60CkE+bkqXREREZLd491wb5qV2Ro8gbxwuKkN2vhajI0OULomIiMhuWT2m6UaKi4tteThqATER9eOadMoWQkREZOcsDk0eHh64cOGCvDx69GgUFhbKy+fPn0dICHsqHE10mC8APk6FiIjoZiwOTVVVVRBCyMu//vorKisrzdpcu50cQ31P074zetQYTQpXQ0REZL9senmOA8EdT+d2ntC4u8BQa0JuIZ8dSERE1BSbhiZyPCqVJE89wEkuiYiImmZxaJIkyawn6fplclzRYZzkkoiI6GYsnnJACIEePXrIQam8vBzR0dFQqVTydnJMMRG+ANjTREREdCMWh6aVK1c2Zx2koH5hvpAkIP/SZVwsNyDAS610SURERHbH4tA0bdq05qyDFOTj5oJu7b1wrLgc2fk6PNA7SOmSiIiI7A4HghMAIObKw3s5XxMREVHjGJoIwNWH93IwOBERUeMYmgiA+SSXtZzkkoiIqAFFQ9P27dsxfvx4hIaGQpIkrFu3Tt5WU1ODF154AVFRUfD09ERoaCieeOIJnDt3zuwYw4cPl6c/qH899thjZm20Wi3i4+Oh0Wig0WgQHx8PnU5n1iY/Px/jx4+Hp6cnAgICMGfOHFRXVzfXqdudbu294K12xuVqI46cL1O6HCIiIrujaGiqqKhAv379sGzZsgbbLl++jKysLPz1r39FVlYW1qxZg6NHj2LChAkN2iYkJKCwsFB+LV++3Gz7lClTkJOTg5SUFKSkpCAnJwfx8fHydqPRiLFjx6KiogI7duxAcnIyVq9ejcTERNuftJ1SqST05ySXRERETbL47rl6RqMRn332GTZv3ozi4mKYTOaXcrZs2WLxscaMGYMxY8Y0uk2j0SA1NdVs3fvvv4+77roL+fn5CA8Pl9d7eHggODi40ePk5uYiJSUFGRkZGDRoEADgk08+QWxsLI4cOYKePXti06ZNOHToEAoKChAaGgoAePvttzF9+nS8/vrr8PHxafTYBoMBBoNBXi4tdezHkESH+eKXYxeRla/F7wdHKF0OERGRXbG6p2nu3LmYO3cujEYjIiMj0a9fP7NXc9Lr9ZAkCb6+vmbrV61ahYCAAPTp0wcLFixAWdnVy0vp6enQaDRyYAKAwYMHQ6PRYOfOnXKbyMhIOTABwKhRo2AwGJCZmdlkPUlJSfIlP41Gg7CwMBudqTKir9xBl8OeJiIiogas7mlKTk7Gf/7zHzz44IPNUU+Tqqqq8OKLL2LKlClmPT9Tp05F586dERwcjAMHDmDhwoX47bff5F6qoqIiBAYGNjheYGAgioqK5DZBQeZzE/n5+cHV1VVu05iFCxdi/vz58nJpaalDB6f+Yb4AgJMXK6CtqIafp6uyBREREdkRq0OTq6srunXr1hy1NKmmpgaPPfYYTCYTPvzwQ7NtCQkJ8s+RkZHo3r07Bg4ciKysLMTExABAo8/IE0I0eJbezdpcT61WQ61uPbNn+3m6okuAJ05erEBOgQ733dEwbBIREbVVVl+eS0xMxD//+c8We9ZcTU0NJk2ahLy8PKSmpjY5vqheTEwMXFxccOzYMQBAcHAwzp8/36DdhQsX5N6l4ODgBj1KWq0WNTU1DXqgWrtoTnJJRETUKIt6mh5++GGz5S1btuCnn35Cnz594OLiYrZtzZo1NiuuPjAdO3YMW7duRbt27W66z8GDB1FTU4OQkBAAQGxsLPR6PXbv3o277roLALBr1y7o9XoMGTJEbvP666+jsLBQ3m/Tpk1Qq9UYMGCAzc7HEUSH+2J11hlkcVwTERGRGYtCk0ajMVv+3e9+Z5M3Ly8vx/Hjx+XlvLw85OTkwN/fH6Ghofi///s/ZGVlYf369TAajXJvkL+/P1xdXXHixAmsWrUKDz74IAICAnDo0CEkJiYiOjoaQ4cOBQD06tULo0ePRkJCgjwVwdNPP41x48ahZ8+eAIC4uDj07t0b8fHxeOutt3Dp0iUsWLAACQkJN+3Zam3qH6eSU6CD0STgpGr68iQREVGbIhS0detWAaDBa9q0aSIvL6/RbQDE1q1bhRBC5Ofni2HDhgl/f3/h6uoqunbtKubMmSNKSkrM3qekpERMnTpVeHt7C29vbzF16lSh1WrN2pw+fVqMHTtWuLu7C39/fzFr1ixRVVVl1fno9XoBQOj1+tv5WBRVU2sUvf76k4h4Yb04XFiqdDlERETNztLvb0kI6wYnjRgxAmvWrGlw239paSkmTpxo1TxNrU1paSk0Gg30er1D91A9/nEG0k+WYOnDUXjsrvCb70BEROTALP3+tnog+LZt2xp9vEhVVRV++eUXaw9HdogP7yUiImrI4ikH9u3bJ/986NAhs7vNjEYjUlJS0KFDB9tWR4q4egedTtlCiIiI7IjFoal///7yA3FHjBjRYLu7uzvef/99mxZHyqjvaTpWXA59ZQ007i433oGIiKgNsDg05eXlQQiBLl26YPfu3Wjfvr28zdXVFYGBgXBycmqWIqllBXipEe7vgfxLl/FbgQ7DerS/+U5EREStnMWhKSKi7gGu1z+gl1qnmHBf5F+6jOx8hiYiIiLAwtD0/fffY8yYMXBxccH3339/w7YTJkywSWGkrOhwP6zLOcfB4ERERFdYFJomTpwoP/h24sSJTbaTJAlGo9FWtZGCrp3k0mQSUHGSSyIiauMsCk3XXpLj5bm24Y4Qb7i5qKCvrMHJixXoFuildElERESKsnqepsuXLzdHHWRnXJxU6NvBFwAf3ktERATcQmjy9fXFkCFD8NJLL2Hjxo2oqKhojrrIDlyd5FKnaB1ERET2wOrQlJaWhgkTJiArKwuPPvoo/Pz8MHjwYLz44ov46aefmqNGUkh9aGJPExEREWD1s+euZTQasWfPHnz00UdYtWoVTCZTmx4I3lqePVfvfGkVBi3ZDJUE7Fs8Cl5qi2eoICIichiWfn/f0rfg4cOHsW3bNqSlpWHbtm2oqanB+PHjce+9995ywWR/gnzc0MHXHWd1ldhXoMOQbgFKl0RERKQYq0NTcHAwampqMGLECAwfPhwvvfQSoqKimqM2sgPR4b44q6tENkMTERG1cVaPaQoODkZ5eTny8/ORn5+PM2fOoLy8vDlqIztQ//DerNMc10RERG2b1aEpJycH58+fx1/+8hfU1tbir3/9K9q3b49BgwbhxRdfbI4aSUEx9YPBC3S4jeFvREREDu+2BoJfunQJ27Ztw3fffYevv/6aA8Fb2UBwADDUGhG1aBOqjSZsWzAcnQI8lS6JiIjIpiz9/ra6p2nt2rWYO3cu+vXrh8DAQPzpT39CRUUF/vGPf2Dfvn23VTTZH7WzE/p0qPsFyi7gJToiImq7rB4I/sc//hHDhg1DQkIChg8fjsjIyOaoi+xITLgfsvN1yDqtw++iOypdDhERkSKsDk3FxcXNUQfZMXmSS/Y0ERFRG2b15Tlqe2Ku3EGXW1iGyuq2O2aNiIjaNoYmuqkQjRuCfNQwmgT2ndEpXQ4REZEiGJropiRJknubsgt0yhZDRESkEIYmskj9uCZOcklERG3VbYem0tJSrFu3Drm5ubaoh+zUtT1NnOSSiIjaIqtD06RJk7Bs2TIAQGVlJQYOHIhJkyahb9++WL16tc0LJPsQ2UEDZ5WEC2UGnNFWKl0OERFRi7M6NG3fvh333HMPgLqJLoUQ0Ol0eO+99/Daa6/ZvECyD24uTugdWj/JpU7ZYoiIiBRgdWjS6/Xw9/cHAKSkpOCRRx6Bh4cHxo4di2PHjtm8QLIf8iW6fI5rIiKitsfq0BQWFob09HRUVFQgJSUFcXFxAACtVgs3NzebF0j2Qx4Mnq9TtA4iIiIlWD0j+Lx58zB16lR4eXkhPDwcw4cPB1B32S4qKsrW9ZEdqe9pOnROj6oaI9xcnBSuiIiIqOVYHZqeffZZ3HXXXSgoKMADDzwAlaqus6pLly4c09TKdfRzR4CXKy6WV+PgOT0GRPgrXRIREVGLuaUpBwYOHIixY8fi7NmzqK2tBQCMHTsWQ4cOtWlxZF8kSUK0PK5Jp2wxRERELczq0HT58mU8+eST8PDwQJ8+fZCfnw8AmDNnDpYuXWrVsbZv347x48cjNDQUkiRh3bp1ZtuFEFi8eDFCQ0Ph7u6O4cOH4+DBg2ZtDAYDZs+ejYCAAHh6emLChAk4c+aMWRutVov4+HhoNBpoNBrEx8dDp9OZtcnPz8f48ePh6emJgIAAzJkzB9XV1VadT1twdVwTB4MTEVHbYnVoWrhwIX777Tds27bNbOD3yJEj8e2331p1rIqKCvTr10+e9+l6b775Jt555x0sW7YMe/bsQXBwMB544AGUlZXJbebNm4e1a9ciOTkZO3bsQHl5OcaNGwej8eqDZadMmYKcnBykpKQgJSUFOTk5iI+Pl7cbjUaMHTsWFRUV2LFjB5KTk7F69WokJiZadT5tQQx7moiIqK0SVgoPDxfp6elCCCG8vLzEiRMnhBBCHDt2THh7e1t7OBkAsXbtWnnZZDKJ4OBgsXTpUnldVVWV0Gg04qOPPhJCCKHT6YSLi4tITk6W25w9e1aoVCqRkpIihBDi0KFDAoDIyMiQ26SnpwsA4vDhw0IIITZs2CBUKpU4e/as3Oabb74RarVa6PX6JmuuqqoSer1efhUUFAgAN9zH0VUYakTnF9eLiBfWi3O6y0qXQ0REdNv0er1F399W9zRduHABgYGBDdZXVFRAkqTbzXCyvLw8FBUVyVMaAIBarca9996LnTt3AgAyMzNRU1Nj1iY0NBSRkZFym/T0dGg0GgwaNEhuM3jwYGg0GrM2kZGRCA0NlduMGjUKBoMBmZmZTdaYlJQkX/LTaDQICwuzzcnbMQ9XZ9wRfGWSS/Y2ERFRG2J1aLrzzjvx448/ysv1QemTTz5BbGyszQorKioCAAQFBZmtDwoKkrcVFRXB1dUVfn5+N2zTWMgLDAw0a3P9+/j5+cHV1VVu05iFCxdCr9fLr4KCAivP0jHFRPgC4CSXRETUtlg95UBSUhJGjx6NQ4cOoba2Fv/85z9x8OBBpKenIy0tzeYFXt97JYS4aY/W9W0aa38rba6nVquhVqtvWEtrFB3mh68y8jnJJRERtSlW9zQNGTIEv/76Ky5fvoyuXbti06ZNCAoKQnp6OgYMGGCzwoKDgwGgQU9PcXGx3CsUHByM6upqaLXaG7Y5f/58g+NfuHDBrM3176PValFTU9OgB4qAmIi6nr39Z/WorjUpXA0REVHLuKV5mqKiovD555/jwIEDOHToEL766iubzwbeuXNnBAcHIzU1VV5XXV2NtLQ0DBkyBAAwYMAAuLi4mLUpLCzEgQMH5DaxsbHQ6/XYvXu33GbXrl3Q6/VmbQ4cOIDCwkK5zaZNm6BWq20aBFuLTu084OfhgupaEw4VlipdDhERUYuw+vIcAJhMJhw/fhzFxcUwmcx7GoYNG2bxccrLy3H8+HF5OS8vDzk5OfD390d4eDjmzZuHJUuWoHv37ujevTuWLFkCDw8PTJkyBQCg0Wjw5JNPIjExEe3atYO/vz8WLFiAqKgojBw5EgDQq1cvjB49GgkJCVi+fDkA4Omnn8a4cePQs2dPAEBcXBx69+6N+Ph4vPXWW7h06RIWLFiAhIQE+Pj43MpH1KrVT3K55XAxsvO16B/mq3RJREREzc/a2/LS09NF586dhUqlEpIkmb1UKpVVx9q6dasA0OA1bdo0IUTdtAOLFi0SwcHBQq1Wi2HDhon9+/ebHaOyslLMmjVL+Pv7C3d3dzFu3DiRn59v1qakpERMnTpVeHt7C29vbzF16lSh1WrN2pw+fVqMHTtWuLu7C39/fzFr1ixRVVVl1flYestia/Dez0dFxAvrxayvs5QuhYiI6LZY+v0tCSGENSGrf//+6NGjB15++WWEhIQ0GCit0Whsk+YcUGlpKTQaDfR6favvodpx7CJ+v2IXOvq5Y8cLI5Quh4iI6JZZ+v1t9eW5Y8eO4X//+x+6det2WwWSY+sXpoEkAWe0lSguq0Kgt9vNdyIiInJgVg8EHzRokNk4JGqbvN1c0CPQGwAnuSQiorbB6p6m2bNnIzExEUVFRYiKioKLi4vZ9r59+9qsOLJvMRG+OHK+DNn5OozqE6x0OURERM3K6tD0yCOPAABmzJghr5MkSZ4I8toH5VLrFh3mh292FyCLM4MTEVEbYHVoysvLa446yAHVP05l3xkdao0mODvd0rRfREREDsHq0BQREdEcdZAD6hLgBR83Z5RW1eJwURkiO7TdOyeJiKj1syg0ff/99xgzZgxcXFzw/fff37DthAkTbFIY2T+VSkL/cD9sP3oB2flahiYiImrVLApNEydORFFREQIDAzFx4sQm23FMU9sTHeaL7UcvICtfh/hYpashIiJqPhaFpmsflXL9Y1OobYsO9wUAZHMwOBERtXI2G7lbUFBgdkcdtQ3RYX4AgFMll3GpolrhaoiIiJqPzULTpUuX8Pnnn9vqcOQgNB4u6NreEwB7m4iIqHXjPeJ022LC63qbODM4ERG1ZgxNdNuir4QmTnJJREStGUMT3bb6SS5/K9DBaBLKFkNERNRMLJ7c8uGHH77hdp1Od7u1kIPqHugNL7Uzyg21OHq+DL1CfJQuiYiIyOYsDk0azY0nLtRoNHjiiSduuyByPE4qCf3CNPj1eAmy83UMTURE1CpZHJpWrlzZnHWQg4sO87sSmrSYMihc6XKIiIhsjmOayCbqJ7nkYHAiImqtGJrIJurvoDtxoQL6yzUKV0NERGR7DE1kE/6erujUzgMAkF3A3iYiImp9GJrIZjjJJRERtWYWhaaYmBhotXW9B6+88gouX77crEWRY+K4JiIias0sCk25ubmoqKgAALz88ssoLy9v1qLIMdWPa8op0MHESS6JiKiVsWjKgf79++MPf/gD7r77bggh8Pe//x1eXl6Ntv3b3/5m0wLJcdwR7A13FyeUVdXixIVydA/yVrokIiIim7EoNH322WdYtGgR1q9fD0mS8NNPP8HZueGukiQxNLVhzk4q9O2owa68S8jO1zE0ERFRq2JRaOrZsyeSk5MBACqVCps3b0ZgYGCzFkaOKTrcry40FWgx6c4wpcshIiKyGYtnBK9nMpmaow5qJeTB4Kd1itZBRERka1aHJgA4ceIE3n33XeTm5kKSJPTq1Qtz585F165dbV0fOZj60HS0uAxlVTXwdnNRtiAiIiIbsXqepo0bN6J3797YvXs3+vbti8jISOzatQt9+vRBampqc9RIDiTQ2w0d/dwhBPBbgV7pcoiIiGzG6p6mF198Ec899xyWLl3aYP0LL7yABx54wGbFkWOKCffDGW0lsvO1uLt7gNLlEBER2YTVPU25ubl48sknG6yfMWMGDh06ZJOiyLFxkksiImqNrA5N7du3R05OToP1OTk5vKOOAFzzOJUCHYTgJJdERNQ6WB2aEhIS8PTTT+ONN97AL7/8gh07dmDp0qX44x//iKefftrmBXbq1AmSJDV4zZw5EwAwffr0BtsGDx5sdgyDwYDZs2cjICAAnp6emDBhAs6cOWPWRqvVIj4+HhqNBhqNBvHx8dDpdDY/n7agV4gPXJ1V0F2uQd7FCqXLISIisgmrxzT99a9/hbe3N95++20sXLgQABAaGorFixdjzpw5Ni9wz549MBqN8vKBAwfwwAMP4NFHH5XXjR49GitXrpSXXV1dzY4xb948/PDDD0hOTka7du2QmJiIcePGITMzE05OTgCAKVOm4MyZM0hJSQEAPP3004iPj8cPP/xg83Nq7VydVYjqoEHmaS2y83Xo0r7x2eOJiIgciSRu4/pJWVkZAMDbu+Vmfp43bx7Wr1+PY8eOQZIkTJ8+HTqdDuvWrWu0vV6vR/v27fHll19i8uTJAIBz584hLCwMGzZswKhRo5Cbm4vevXsjIyMDgwYNAgBkZGQgNjYWhw8fRs+ePS2qrbS0FBqNBnq9Hj4+PjY5X0f1+o+H8Mkvefj94HC8NjFK6XKIiIiaZOn3t9WX567l7e3dooGpuroaX331FWbMmAFJkuT127ZtQ2BgIHr06IGEhAQUFxfL2zIzM1FTU4O4uDh5XWhoKCIjI7Fz504AQHp6OjQajRyYAGDw4MHQaDRym8YYDAaUlpaavahO/cN7OcklERG1FrcVmlraunXroNPpMH36dHndmDFjsGrVKmzZsgVvv/029uzZgxEjRsBgMAAAioqK4OrqCj8/P7NjBQUFoaioSG7T2CD2wMBAuU1jkpKS5DFQGo0GYWF8bEi9+sHgh4tKcbm6VuFqiIiIbp9DhaYVK1ZgzJgxCA0NlddNnjwZY8eORWRkJMaPH4+ffvoJR48exY8//njDYwkhzHqrrv25qTbXW7hwIfR6vfwqKCi4hbNqnYI1bgjRuMHESS6JiKiVcJjQdPr0afz888946qmnbtguJCQEEREROHbsGAAgODgY1dXV0GrN5wwqLi5GUFCQ3Ob8+fMNjnXhwgW5TWPUajV8fHzMXnTV1akHOF8TERE5PqtCU01NDe677z4cPXq0uepp0sqVKxEYGIixY8fesF1JSQkKCgoQEhICABgwYABcXFzMHvFSWFiIAwcOYMiQIQCA2NhY6PV67N69W26za9cu6PV6uQ1Zjw/vJSKi1sSqKQdcXFxw4MCBG16yag4mkwkrV67EtGnT4Ox8teTy8nIsXrwYjzzyCEJCQnDq1Cm89NJLCAgIwO9+9zsAgEajwZNPPonExES0a9cO/v7+WLBgAaKiojBy5EgAQK9evTB69GgkJCRg+fLlAOqmHBg3bpzFd85RQ/WDwXMKtDe91ElERGTvrL4898QTT2DFihXNUUuTfv75Z+Tn52PGjBlm652cnLB//3489NBD6NGjB6ZNm4YePXogPT3d7K6+f/zjH5g4cSImTZqEoUOHwsPDAz/88IM8RxMArFq1ClFRUYiLi0NcXBz69u2LL7/8ssXOsTXqE+oDFycJF8urcUZbqXQ5REREt8XqeZpmz56NL774At26dcPAgQPh6elptv2dd96xaYGOhPM0NfTQB7/itwId/vlYfzzUv4PS5RARETVg6fe31TOCHzhwADExMQDQYGwTL7/Q9WLCffFbgQ7Z+TqGJiIicmhWh6atW7c2Rx3USkWH+2Hlr6eQlc876IiIyLHd8pQDx48fx8aNG1FZWTdWhU+zp8bEXLmD7tC5UlTVGG/cmIiIyI5ZHZpKSkpw//33o0ePHnjwwQdRWFgIAHjqqaeQmJho8wLJsXXwdUd7bzVqTQL7z3KSSyIiclxWh6bnnnsOLi4uyM/Ph4eHh7x+8uTJSElJsWlx5PgkSZJ7m7J5iY6IiByY1WOaNm3ahI0bN6Jjx45m67t3747Tp0/brDBqPaLD/bDx4HlOcklERA7N6p6miooKsx6mehcvXoRarbZJUdS61D9OJStfy7FvRETksKwOTcOGDcMXX3whL0uSBJPJhLfeegv33XefTYuj1iGqgwZOKgnFZQYU6quULoeIiOiWWH157q233sLw4cOxd+9eVFdX4/nnn8fBgwdx6dIl/Prrr81RIzk4d1cn9ArxxoGzpcjK1yLU113pkoiIiKxmdU9T7969sW/fPtx111144IEHUFFRgYcffhjZ2dno2rVrc9RIrUD9JbrsfJ2yhRAREd0iq3uaACA4OBgvv/yyrWuhViw63BdfpJ/mJJdEROSwbik0abVarFixArm5uZAkCb169cIf/vAH+Pv727o+aiXqe5oOni2FodYItbPTTfYgIiKyL1ZfnktLS0Pnzp3x3nvvQavV4tKlS3jvvffQuXNnpKWlNUeN1AqE+3vA39MV1UYTDp4rVbocIiIiq1kdmmbOnIlJkyYhLy8Pa9aswZo1a3Dy5Ek89thjmDlzZnPUSK2A+SSXOkVrISIiuhVWh6YTJ04gMTERTk5XL684OTlh/vz5OHHihE2Lo9YlWh4MznFNRETkeKwOTTExMcjNzW2wPjc3F/3797dFTdRKRYf5AmBPExEROSaLBoLv27dP/nnOnDmYO3cujh8/jsGDBwMAMjIy8MEHH2Dp0qXNUyW1Cn3DfKGSgLO6SpwvrUKQj5vSJREREVlMEhY810KlUkGSpJs+AkOSJBiNRpsV52hKS0uh0Wig1+vh4+OjdDl2afS723G4qAwf/T4GoyNDlC6HiIjI4u9vi3qa8vLybFYYtW0xEX44XFSG7HwdQxMRETkUi0JTREREc9dBbUR0mC++3pXPSS6JiMjh3NLklmfPnsWvv/6K4uJimEwms21z5syxSWHUOsVE1N1Bt++MHjVGE1ycrL4XgYiISBFWh6aVK1fimWeegaurK9q1awdJkuRtkiQxNNENdW7nCY27C/SVNcgtLEXfjr5Kl0RERGQRq/83/29/+xv+9re/Qa/X49SpU8jLy5NfJ0+ebI4aqRVRqSREc5JLIiJyQFaHpsuXL+Oxxx6DSsXLKnRrosM4ySURETkeq5PPk08+if/+97/NUQu1EfU9TVnsaSIiIgdi9ZimpKQkjBs3DikpKYiKioKLi4vZ9nfeecdmxVHr1D/cF5IE5F+6jIvlBgR4qZUuiYiI6KasDk1LlizBxo0b0bNnTwBoMBCc6GZ83FzQrb0XjhWXIztfhwd6ByldEhER0U1ZHZreeecdfPrpp5g+fXozlENtRUy435XQpGVoIiIih2D1mCa1Wo2hQ4c2Ry3Uhlwd18TB4ERE5BisDk1z587F+++/3xy1UBty7SSXtUbTTVoTEREpz+rLc7t378aWLVuwfv169OnTp8FA8DVr1tisOGq9urX3grfaGWWGWhw9X47eoXzAMRER2TerQ5Ovry8efvjh5qiF2hCVSkL/cF/8cuwisvK1DE1ERGT3rL48t3Llyhu+bGnx4sWQJMnsFRwcLG8XQmDx4sUIDQ2Fu7s7hg8fjoMHD5odw2AwYPbs2QgICICnpycmTJiAM2fOmLXRarWIj4+HRqOBRqNBfHw8dDqdTc+FGooO8wXAmcGJiMgx2P203n369EFhYaH82r9/v7ztzTffxDvvvINly5Zhz549CA4OxgMPPICysjK5zbx587B27VokJydjx44dKC8vx7hx42A0GuU2U6ZMQU5ODlJSUpCSkoKcnBzEx8e36Hm2RdHhnBmciIgch9WX5zp37nzD+Zhs/fw5Z2dns96lekIIvPvuu/jLX/4iXy78/PPPERQUhK+//hp//OMfodfrsWLFCnz55ZcYOXIkAOCrr75CWFgYfv75Z4waNQq5ublISUlBRkYGBg0aBAD45JNPEBsbiyNHjsjzUZHt9b/S03TyYgW0FdXw83RVtiAiIqIbsDo0zZs3z2y5pqYG2dnZSElJwZ///Gdb1SU7duwYQkNDoVarMWjQICxZsgRdunRBXl4eioqKEBcXJ7dVq9W49957sXPnTvzxj39EZmYmampqzNqEhoYiMjISO3fuxKhRo5Ceng6NRiMHJgAYPHgwNBoNdu7cecPQZDAYYDAY5OXS0lIbn33r5ufpii4Bnjh5sQI5BTrcd0eg0iURERE1yerQNHfu3EbXf/DBB9i7d+9tF3StQYMG4YsvvkCPHj1w/vx5vPbaaxgyZAgOHjyIoqIiAEBQkPnEiEFBQTh9+jQAoKioCK6urvDz82vQpn7/oqIiBAY2/LIODAyU2zQlKSkJL7/88i2fH9Vdojt5sQLZ+VqGJiIisms2G9M0ZswYrF692laHk4/5yCOPICoqCiNHjsSPP/4IoO4yXL3rLxUKIW76OJfr2zTW3pLjLFy4EHq9Xn4VFBTc9JzIHB/eS0REjsJmoel///sf/P39bXW4Rnl6eiIqKgrHjh2Txzld3xtUXFws9z4FBwejuroaWq32hm3Onz/f4L0uXLjQoBfremq1Gj4+PmYvsk7MlcHgOQU6GE1C4WqIiIiaZnVoio6ORkxMjPyKjo5GSEgIXnrpJbz00kvNUaPMYDAgNzcXISEh6Ny5M4KDg5Gamipvr66uRlpaGoYMGQIAGDBgAFxcXMzaFBYW4sCBA3Kb2NhY6PV67N69W26za9cu6PV6uQ01nx5BXvBwdUK5oRYnLpQrXQ4REVGTrB7TNHHiRLNllUqF9u3bY/jw4bjjjjtsVRcAYMGCBRg/fjzCw8NRXFyM1157DaWlpZg2bRokScK8efOwZMkSdO/eHd27d8eSJUvg4eGBKVOmAAA0Gg2efPJJJCYmol27dvD398eCBQvky30A0KtXL4wePRoJCQlYvnw5AODpp5/GuHHjeOdcC3B2UqFvRw0yTl5C1mktegR5K10SERFRo6wOTYsWLWqOOhp15swZPP7447h48SLat2+PwYMHIyMjAxEREQCA559/HpWVlXj22Weh1WoxaNAgbNq0Cd7eV794//GPf8DZ2RmTJk1CZWUl7r//fnz22WdwcnKS26xatQpz5syR77KbMGECli1b1mLn2dbFhPsh4+QlZOfr8Nhd4UqXQ0RE1ChJCMGBJDZSWloKjUYDvV7P8U1WSD10Hglf7EX3QC+kzr9X6XKIiKiNsfT72+KeJpVKddO7ySRJQm1treVVEuHqHXTHisuhr6yBxt3lxjsQEREpwOLQtHbt2ia37dy5E++//z7YaUW3IsBLjXB/D+RfuozfCnQY1qO90iURERE1YHFoeuihhxqsO3z4MBYuXIgffvgBU6dOxauvvmrT4qjtiAn3Rf6ly8jOZ2giIiL7dEvzNJ07dw4JCQno27cvamtrkZOTg88//xzh4RzES7em/uG9WXx4LxER2SmrQpNer8cLL7yAbt264eDBg9i8eTN++OEHREZGNld91EZcO8mliZNcEhGRHbI4NL355pvo0qUL1q9fj2+++QY7d+7EPffc05y1URtyR4g31M4q6CtrkFdSoXQ5REREDVg85YBKpYK7uztGjhxpNsfR9dasWWOz4hwNpxy4PY9+tBN7Tmnx1v/1xaMDw5Quh4iI2gibTznwxBNP3HTKAaLbERPuhz2ntMgu0DE0ERGR3bE4NH322WfNWAbR1fmask5zMDgREdmfW7p7jqg51N9Bd/R8GcoNnCSViIjsC0MT2Y0gHzd08HWHSQD7CnRKl0NERGSGoYnsSv0lumyGJiIisjMMTWRX6i/RZXOSSyIisjMMTWRXYuoHg+fr+CxDIiKyKwxNZFd6h/rA1UmFSxXVyL90WelyiIiIZAxNZFfUzk7o06FuYjE+h46IiOwJQxPZnRh5XJNO2UKIiIiuwdBEdkee5JI9TUREZEcYmsju1Pc05RaWobLaqHA1REREdRiayO6EaNwQ5KOG0SSw74xO6XKIiIgAMDSRHZIk6eq4Jk5ySUREdoKhieySPDM4xzUREZGdYGgiu1Q/MzgnuSQiInvB0ER2KaqDBs4qCRfKDDirq1S6HCIiIoYmsk9uLk7oHVo/yaVO2WKIiIjA0ER2LIYP7yUiIjvC0ER2K/qah/cSEREpjaGJ7FZ9T9Ohc3pU1XCSSyIiUhZDE9mtjn7uCPByRY1R4OC5UqXLISKiNo6hieyWJEny1AMc10REREpjaCK7dnWSS52idRARETE0kV2LDquf5JI9TUREpCy7Dk1JSUm488474e3tjcDAQEycOBFHjhwxazN9+nRIkmT2Gjx4sFkbg8GA2bNnIyAgAJ6enpgwYQLOnDlj1kar1SI+Ph4ajQYajQbx8fHQ6XTNfYp0E/3CNFBJQKG+CoV6TnJJRETKsevQlJaWhpkzZyIjIwOpqamora1FXFwcKioqzNqNHj0ahYWF8mvDhg1m2+fNm4e1a9ciOTkZO3bsQHl5OcaNGwej8eodWVOmTEFOTg5SUlKQkpKCnJwcxMfHt8h5UtM8XJ1xR3DdJJe8REdEREpyVrqAG0lJSTFbXrlyJQIDA5GZmYlhw4bJ69VqNYKDgxs9hl6vx4oVK/Dll19i5MiRAICvvvoKYWFh+PnnnzFq1Cjk5uYiJSUFGRkZGDRoEADgk08+QWxsLI4cOYKePXs20xmSJWIifHGosBRfpJ/CHcHe6NLeS+mSiIioDbLrnqbr6fV6AIC/v7/Z+m3btiEwMBA9evRAQkICiouL5W2ZmZmoqalBXFycvC40NBSRkZHYuXMnACA9PR0ajUYOTAAwePBgaDQauU1jDAYDSktLzV5ke+P6hkKSgIyTlzDynTTMTc7GsfNlSpdFRERtjMOEJiEE5s+fj7vvvhuRkZHy+jFjxmDVqlXYsmUL3n77bezZswcjRoyAwWAAABQVFcHV1RV+fn5mxwsKCkJRUZHcJjAwsMF7BgYGym0ak5SUJI+B0mg0CAsLs8Wp0nUGd2mHdc8OxchegTAJ4Lucc4h7dztmfp2Fw0UMqkRE1DLs+vLctWbNmoV9+/Zhx44dZusnT54s/xwZGYmBAwciIiICP/74Ix5++OEmjyeEgCRJ8vK1PzfV5noLFy7E/Pnz5eXS0lIGp2bSL8wX/552Jw6c1eP9Lcew8eB5/LivED/uK8SoPkGYPaI7IjtolC6TiIhaMYfoaZo9eza+//57bN26FR07drxh25CQEERERODYsWMAgODgYFRXV0OrNb9lvbi4GEFBQXKb8+fPNzjWhQsX5DaNUavV8PHxMXtR84rsoMHy+IFImXcPxvYNgSQBGw+ex7j3d+Cpz/fgtwKd0iUSEVErZdehSQiBWbNmYc2aNdiyZQs6d+58031KSkpQUFCAkJAQAMCAAQPg4uKC1NRUuU1hYSEOHDiAIUOGAABiY2Oh1+uxe/duuc2uXbug1+vlNmRf7gj2wQdTYrBp3jA81D8UKgn4ObcYD33wK6av3I3M05zXiYiIbEsSQgili2jKs88+i6+//hrfffed2R1sGo0G7u7uKC8vx+LFi/HII48gJCQEp06dwksvvYT8/Hzk5ubC29sbAPCnP/0J69evx2effQZ/f38sWLAAJSUlyMzMhJOTE4C6sVHnzp3D8uXLAQBPP/00IiIi8MMPP1hcb2lpKTQaDfR6PXudWtjJC+X4YOsJrMs5C6Op7lf67m4BmHN/d9zV2f8mexMRUVtm6fe3XYempsYTrVy5EtOnT0dlZSUmTpyI7Oxs6HQ6hISE4L777sOrr75qNraoqqoKf/7zn/H111+jsrIS999/Pz788EOzNpcuXcKcOXPw/fffAwAmTJiAZcuWwdfX1+J6GZqUd7qkAh9uPYHVWWdQeyU8De7ijzn3d0dsl3Y3HKNGRERtU6sITY6Gocl+FFy6jI/STuA/ewtQY6z7FR8Y4Yc593fHPd0DGJ6IiEjG0KQAhib7c05XieVpJ/DNngJU15oAAP3DfDH3/u4Y3rM9wxMRETE0KYGhyX6dL63C8rSTWLXrNAxXwlNUBw1mj+iGB3oHMTwREbVhDE0KYGiyf8VlVfj3L3n4Mv00Kmvqnj3YK8QHc0Z0w6g+wVCpGJ6IiNoahiYFMDQ5jpJyA1bsyMPnO0+horouPPUI8sKsEd0xNioETgxPRERtBkOTAhiaHI/ucjU+3ZGHlb+eQpmhFgDQpb0nZo/ohvF9Q+HsZNdTmRERkQ0wNCmAoclx6Str8PnOU1ixIw/6yhoAQKd2Hph5XzdMjO4AF4YnIqJWi6FJAQxNjq+sqgZfpJ/Gv385Ce3luvAU5u+OmcO74eGYjnB1ZngiImptGJoUwNDUelQYavFVxml88stJXCyvBgB08HXHM8O7YtLAjlA7OylcIRER2QpDkwIYmlqfymojvt6dj4/STuBCmQEAEOzjhmfu7YLH7gqHmwvDExGRo2NoUgBDU+tVVWPEt3sK8FHaCRTqqwAA7b3V+OOwLpgyKBwers4KV0hERLeKoUkBDE2tn6HWiP9lnsGHW0/grK4SANDO0xUJw7ogfnAEPNUMT0REjoahSQEMTW1Hda0Ja7PPYNnW4yi4VBee/Dxc8NQ9XfBEbAS83VwUrpCIiCzF0KQAhqa2p8Zownc55/DB1uPIu1gBAPBxc8aMuzvjD0M6Q+PB8EREZO8YmhTA0NR21RpNWL+vEO9vOYYTF+rCk7faGdOHdsKMoZ3h5+mqcIVERNQUhiYFMDSR0STw04FCvL/5OI6cLwMAeLo6IT62ExLu6Yx2XmqFKyQiousxNCmAoYnqmUwCmw4V4b3Nx3GosBQA4O7ihN8PDkfCsC4I9HZTuEIiIqrH0KQAhia6nhACm3OL8d6WY9h3Rg8AUDur8Phd4Xjm3q4I1jA8EREpjaFJAQxN1BQhBLYdvYD3Nh9Ddr4OAODqpMLkO8PwzPCu6ODrrmyBRERtGEOTAhia6GaEEPj1eAn+ufko9pzSAgBcnCT834AwPDu8K8L8PRSukIio7WFoUgBDE1lKCIGMk5fw3uZjSD9ZAgBwUkl4OLoDZt7XDZ0CPBWukIio7WBoUgBDE92KPafqwtMvxy4CAFQSMLF/B8wc0Q1d23spXB0RUevH0KQAhia6HVn5Wry/+Ri2HrkAAJAkYFzfUMwe0Q09grwVro6IqPViaFIAQxPZwr4zOry3+Th+zj0vr3swKhi/HxyB7oHeCPByhSRJClZIRNS6MDQpgKGJbOnAWT2WbTmOlINFZuu91M7oFOCBTu080TnAE53aeaJTQN3Pfh4uDFRERFZiaFIAQxM1h8NFpfho2wnsOaXFOX0lbvRfrI+bc12QuhKirg1VGnc+B4+IqDEMTQpgaKLmVlVjRMGly8i7WIFTJRXIu3gZp678XKivuuG+/p6u6NTOoy5QXdM71SnAE15q5xY6AyIi+2Pp9zf/pSRyIG4uTuge5I3ujQwMr6w24vSlCpy6WIGTF+v+PHXxMvJKKnChzIBLFdW4VFGNrCuTa14rwEuNzlcu+XUy66HygIcr/5kgIgLY02RT7Gkie1VuqJV7pE5dvNJDdeXnkorqG+4b5KO+On7qSpjqHOCJiHYecHNxaqEzICJqPrw8pwCGJnJEpVU1V4JUXc9U3WW/uoClu1zT5H6SBIRq3BsdlB7u7wFXZ1ULngUR0a1jaFIAQxO1NrrL1Y2On8q7WIGyqtom91NJQAc/d7MwVd9T1dHPHS5ODFREZD8YmhTA0ERthRACJRXVV3uoSq6Mn7ry8+VqY5P7OqskdPRzN7vUVz84vYOfO5xUnDKBiFoWB4ITUbORJAkBXmoEeKkxsJO/2TYhBC6UGRrtoTpVUoGqGhNOlVzGqZLLAC6Y7eviJCHM3wOdrw1TV/4M8XGDioGKiBTE0HSdDz/8EG+99RYKCwvRp08fvPvuu7jnnnuULovIYUiShEAfNwT6uGFQl3Zm20wmgfNlVQ3HT12swOlLl1Fda8LJCxU4eaGiwXFdnCS4uzjBzcUJahcV3JzrfnZzUdWtc776s1uj21VQX9nfzbm+3TX7XNlffWWdq5OKE4USkRmGpmt8++23mDdvHj788EMMHToUy5cvx5gxY3Do0CGEh4crXR6Rw1OpJIRo3BGicceQrubbjCaBQn2lPE3CqSthKq+kAvkll1FjFKgx1qL0BmOpbEmSALWzeaCqC2zXhq5GAte14ayR8ObmoroS8K79uW4bx3oR2TeOabrGoEGDEBMTg3/961/yul69emHixIlISkq66f4c00TUPGqNJhSXGXC52oiqGiMMtSYYaoyoqjWiqsaEqppr/ryyzlBzta28vdZo1vbqtqvblfwX0UklmfWCXe1Rq1vnpJKgkiRIEur+RF3PXt0yIEGCSlX3pyTVbatbX9ce1+xXf5wG+1+z7uryTfa/ZvmG+5u1q9sGucarbRses+5P4Oq2+uNfKevKNgDXrDNrW78/6jY22H7lKPLxrju+dG0NN3mv+ve5WvO1fyfm25s61rXvc/2x2rogHzeb/w8GxzRZqbq6GpmZmXjxxRfN1sfFxWHnzp2N7mMwGGAwGOTl0tLSZq2RqK1ydlIh1Ne92d9HCIEao5DDlaHGssDVIHzVGFF13XbDDY5Tz2gSqKg2ouIGA+mJ2rotifeiS3svRd6boemKixcvwmg0IigoyGx9UFAQioqKGt0nKSkJL7/8ckuUR0QtQJIkuDpLcHVWwcetZZ7VJ4Qw6w0zmPWemYcvo0lACMAkBMSVfeuWAQEBU93KuuX6P83a1e1nurJ8w/1Rf5wr7VA3Jq3B/tcsmwQACJhMDfe/+t5X31c00db8mHVdf1ffs669uHJuuLJc91lec77XLOOa5bpdrj1WXfur+4tr2l09Fho99jX71e9z3Xa5Bvn9ReN11r/PDd6L14XqKDnWkKHpOtf/ZQghmvwLWrhwIebPny8vl5aWIiwsrFnrI6LWRZIk+XIcEdk3hqYrAgIC4OTk1KBXqbi4uEHvUz21Wg21Wt0S5REREZHCeKvGFa6urhgwYABSU1PN1qempmLIkCEKVUVERET2gj1N15g/fz7i4+MxcOBAxMbG4uOPP0Z+fj6eeeYZpUsjIiIihTE0XWPy5MkoKSnBK6+8gsLCQkRGRmLDhg2IiIhQujQiIiJSGOdpsiHO00REROR4LP3+5pgmIiIiIgswNBERERFZgKGJiIiIyAIMTUREREQWYGgiIiIisgBDExEREZEFGJqIiIiILMDQRERERGQBhiYiIiIiC/AxKjZUP7l6aWmpwpUQERGRpeq/t2/2kBSGJhsqKysDAISFhSlcCREREVmrrKwMGo2mye189pwNmUwmnDt3Dt7e3pAkyWbHLS0tRVhYGAoKCtrsM+3a+mfA82/b5w/wM2jr5w/wM2jO8xdCoKysDKGhoVCpmh65xJ4mG1KpVOjYsWOzHd/Hx6dN/odyrbb+GfD82/b5A/wM2vr5A/wMmuv8b9TDVI8DwYmIiIgswNBEREREZAGGJgegVquxaNEiqNVqpUtRTFv/DHj+bfv8AX4Gbf38AX4G9nD+HAhOREREZAH2NBERERFZgKGJiIiIyAIMTUREREQWYGgiIiIisgBDkwP48MMP0blzZ7i5uWHAgAH45ZdflC6pxWzfvh3jx49HaGgoJEnCunXrlC6pRSUlJeHOO++Et7c3AgMDMXHiRBw5ckTpslrMv/71L/Tt21eezC42NhY//fST0mUpJikpCZIkYd68eUqX0mIWL14MSZLMXsHBwUqX1aLOnj2L3//+92jXrh08PDzQv39/ZGZmKl1Wi+nUqVOD3wFJkjBz5swWr4Whyc59++23mDdvHv7yl78gOzsb99xzD8aMGYP8/HylS2sRFRUV6NevH5YtW6Z0KYpIS0vDzJkzkZGRgdTUVNTW1iIuLg4VFRVKl9YiOnbsiKVLl2Lv3r3Yu3cvRowYgYceeggHDx5UurQWt2fPHnz88cfo27ev0qW0uD59+qCwsFB+7d+/X+mSWoxWq8XQoUPh4uKCn376CYcOHcLbb78NX19fpUtrMXv27DH7+09NTQUAPProoy1fjCC7dtddd4lnnnnGbN0dd9whXnzxRYUqUg4AsXbtWqXLUFRxcbEAINLS0pQuRTF+fn7i3//+t9JltKiysjLRvXt3kZqaKu69914xd+5cpUtqMYsWLRL9+vVTugzFvPDCC+Luu+9Wugy7MnfuXNG1a1dhMpla/L3Z02THqqurkZmZibi4OLP1cXFx2Llzp0JVkZL0ej0AwN/fX+FKWp7RaERycjIqKioQGxurdDktaubMmRg7dixGjhypdCmKOHbsGEJDQ9G5c2c89thjOHnypNIltZjvv/8eAwcOxKOPPorAwEBER0fjk08+UbosxVRXV+Orr77CjBkzIElSi78/Q5Mdu3jxIoxGI4KCgszWBwUFoaioSKGqSClCCMyfPx933303IiMjlS6nxezfvx9eXl5Qq9V45plnsHbtWvTu3VvpslpMcnIysrKykJSUpHQpihg0aBC++OILbNy4EZ988gmKioowZMgQlJSUKF1aizh58iT+9a9/oXv37ti4cSOeeeYZzJkzB1988YXSpSli3bp10Ol0mD59uiLv76zIu5JVrk/TQghFEjYpa9asWdi3bx927NihdCktqmfPnsjJyYFOp8Pq1asxbdo0pKWltYngVFBQgLlz52LTpk1wc3NTuhxFjBkzRv45KioKsbGx6Nq1Kz7//HPMnz9fwcpahslkwsCBA7FkyRIAQHR0NA4ePIh//etfeOKJJxSuruWtWLECY8aMQWhoqCLvz54mOxYQEAAnJ6cGvUrFxcUNep+odZs9eza+//57bN26FR07dlS6nBbl6uqKbt26YeDAgUhKSkK/fv3wz3/+U+myWkRmZiaKi4sxYMAAODs7w9nZGWlpaXjvvffg7OwMo9GodIktztPTE1FRUTh27JjSpbSIkJCQBv+D0KtXrzZzM9C1Tp8+jZ9//hlPPfWUYjUwNNkxV1dXDBgwQL5ToF5qaiqGDBmiUFXUkoQQmDVrFtasWYMtW7agc+fOSpekOCEEDAaD0mW0iPvvvx/79+9HTk6O/Bo4cCCmTp2KnJwcODk5KV1iizMYDMjNzUVISIjSpbSIoUOHNphm5OjRo4iIiFCoIuWsXLkSgYGBGDt2rGI18PKcnZs/fz7i4+MxcOBAxMbG4uOPP0Z+fj6eeeYZpUtrEeXl5Th+/Li8nJeXh5ycHPj7+yM8PFzBylrGzJkz8fXXX+O7776Dt7e33Ouo0Wjg7u6ucHXN76WXXsKYMWMQFhaGsrIyJCcnY9u2bUhJSVG6tBbh7e3dYPyap6cn2rVr12bGtS1YsADjx49HeHg4iouL8dprr6G0tBTTpk1TurQW8dxzz2HIkCFYsmQJJk2ahN27d+Pjjz/Gxx9/rHRpLcpkMmHlypWYNm0anJ0VjC4tfr8eWe2DDz4QERERwtXVVcTExLSp2823bt0qADR4TZs2TenSWkRj5w5ArFy5UunSWsSMGTPk3/327duL+++/X2zatEnpshTV1qYcmDx5sggJCREuLi4iNDRUPPzww+LgwYNKl9WifvjhBxEZGSnUarW44447xMcff6x0SS1u48aNAoA4cuSIonVIQgihTFwjIiIichwc00RERERkAYYmIiIiIgswNBERERFZgKGJiIiIyAIMTUREREQWYGgiIiIisgBDExEREZEFGJqIiIiILMDQRERNOnXqFCRJQk5OjtKlyA4fPozBgwfDzc0N/fv3V7ocasJnn30GX19feXnx4sX8+yKHx9BEZMemT58OSZKwdOlSs/Xr1q2DJEkKVaWsRYsWwdPTE0eOHMHmzZubbFdUVIS5c+eiW7ducHNzQ1BQEO6++2589NFHuHz5coP2S5YsgZOTU4PPmmxjwYIFN/z7InIEDE1Eds7NzQ1vvPEGtFqt0qXYTHV19S3ve+LECdx9992IiIhAu3btGm1z8uRJREdHY9OmTViyZAmys7Px888/47nnnsMPP/yAn3/+ucE+K1euxPPPP49PP/30lmu7kabOuaam5paOd6v72ZqldXh5eTX590XkKBiaiOzcyJEjERwcjKSkpCbbNHbp491330WnTp3k5enTp2PixIlYsmQJgoKC4Ovri5dffhm1tbX485//DH9/f3Ts2LHR0HD48GEMGTIEbm5u6NOnD7Zt22a2/dChQ3jwwQfh5eWFoKAgxMfH4+LFi/L24cOHY9asWZg/fz4CAgLwwAMPNHoeJpMJr7zyCjp27Ai1Wo3+/fsjJSVF3i5JEjIzM/HKK69AkiQsXry40eM8++yzcHZ2xt69ezFp0iT06tULUVFReOSRR/Djjz9i/PjxZu3T0tJQWVmJV155BRUVFdi+fXujx73W2bNnMXnyZPj5+aFdu3Z46KGHcOrUKXl7/eedlJSE0NBQ9OjRQ77c+Z///AfDhw+Hm5sbvvrqq5ued1P7nT59GuPHj4efnx88PT3Rp08fbNiwocmaO3XqhFdffRVTpkyBl5cXQkND8f7775u10ev1ePrppxEYGAgfHx+MGDECv/32m7y9/nft008/RZcuXaBWq2HJI0yv/x2t/3z+/ve/IyQkBO3atcPMmTPNQlh1dTWef/55dOjQAZ6enhg0aFCD3z2ilsTQRGTnnJycsGTJErz//vs4c+bMbR1ry5YtOHfuHLZv34533nkHixcvxrhx4+Dn54ddu3bhmWeewTPPPIOCggKz/f785z8jMTER2dnZGDJkCCZMmICSkhIAQGFhIe699170798fe/fuRUpKCs6fP49JkyaZHePzzz+Hs7Mzfv31VyxfvrzR+v75z3/i7bffxt///nfs27cPo0aNwoQJE3Ds2DH5vfr06YPExEQUFhZiwYIFDY5RUlKCTZs2YebMmfD09Gz0fa6/tLlixQo8/vjjcHFxweOPP44VK1bc8HO8fPky7rvvPnh5eWH79u3YsWMHvLy8MHr0aLMepc2bNyM3NxepqalYv369vP6FF17AnDlzkJubi1GjRt30vJvab+bMmTAYDNi+fTv279+PN954A15eXjes/a233kLfvn2RlZWFhQsX4rnnnkNqaioAQAiBsWPHoqioCBs2bEBmZiZiYmJw//3349KlS/Ixjh8/jv/85z9YvXr1bY1327p1K06cOIGtW7fi888/x2effYbPPvtM3v6HP/wBv/76K5KTk7Fv3z48+uijGD16dIPPhajFCCKyW9OmTRMPPfSQEEKIwYMHixkzZgghhFi7dq249j/fRYsWiX79+pnt+49//ENERESYHSsiIkIYjUZ5Xc+ePcU999wjL9fW1gpPT0/xzTffCCGEyMvLEwDE0qVL5TY1NTWiY8eO4o033hBCCPHXv/5VxMXFmb13QUGBACCOHDkihBDi3nvvFf3797/p+YaGhorXX3/dbN2dd94pnn32WXm5X79+YtGiRU0eIyMjQwAQa9asMVvfrl074enpKTw9PcXzzz8vr9fr9cLDw0Pk5OQIIYTIzs4WHh4eQq/XN/keK1asED179hQmk0leZzAYhLu7u9i4caMQou7zDgoKEgaDQW5T/3m+++67Vp13U/tFRUWJxYsXN1nn9SIiIsTo0aPN1k2ePFmMGTNGCCHE5s2bhY+Pj6iqqjJr07VrV7F8+XIhRN3vmouLiyguLr7he61cuVJoNBp5+frf0frfx9raWnndo48+KiZPniyEEOL48eNCkiRx9uxZs+Pef//9YuHChZadMJGNsaeJyEG88cYb+Pzzz3Ho0KFbPkafPn2gUl39zz4oKAhRUVHyspOTE9q1a4fi4mKz/WJjY+WfnZ2dMXDgQOTm5gIAMjMzsXXrVnh5ecmvO+64A0Dd+KN6AwcOvGFtpaWlOHfuHIYOHWq2fujQofJ7WeP63qTdu3cjJycHffr0gcFgkNd//fXX6NKlC/r16wcA6N+/P7p06YLk5OQmj52ZmYnjx4/D29tbPmd/f39UVVWZnXNUVBRcXV0b7H/tZ2HNeV//Gc6ZMwevvfYahg4dikWLFmHfvn1N1lzv2r/L+uVr/y7Ly8vRrl07s7/PvLw8s/OKiIhA+/btb/peN9OnTx84OTnJyyEhIfLvXlZWFoQQ6NGjh1ktaWlpZrUQtSRnpQsgIssMGzYMo0aNwksvvYTp06ebbVOpVA3GlTQ2QNfFxcVsWZKkRteZTKab1lMfSkwmE8aPH4833nijQZuQkBD556YulTV13HpCCKvuFOzWrRskScLhw4fN1nfp0gUA4O7ubrb+008/xcGDB+HsfPWfQ5PJhBUrVuDpp59u9D1MJhMGDBiAVatWNdh2bZho6pwbW2/JeV+/31NPPYVRo0bhxx9/xKZNm5CUlIS3334bs2fPbvR9m3Lt32VISEij44aunT7A0r/Lm7nR757JZIKTkxMyMzPNghWAm16CJGouDE1EDmTp0qXo378/evToYba+ffv2KCoqMvuiteXcShkZGRg2bBgAoLa2FpmZmZg1axYAICYmBqtXr0anTp3Mgoe1fHx8EBoaih07dsjvBQA7d+7EXXfdZfFx2rVrhwceeADLli3D7Nmzb/gFv3//fuzduxfbtm2Dv7+/vF6n02HYsGE4cOAAIiMjG+wXExODb7/9Vh4sfTtu97zDwsLksWgLFy7EJ598csPQlJGR0WC5vmcwJiYGRUVFcHZ2NruJQAnR0dEwGo0oLi7GPffco2gtRPV4eY7IgURFRWHq1KkN7ngaPnw4Lly4gDfffBMnTpzABx98gJ9++slm7/vBBx9g7dq1OHz4MGbOnAmtVosZM2YAAGbOnIlLly7h8ccfx+7du3Hy5Els2rQJM2bMgNFotOp9/vznP+ONN97At99+iyNHjuDFF19ETk4O5s6da9VxPvzwQ9TW1mLgwIH49ttvkZubiyNHjuCrr77C4cOH5Z6LFStW4K677sKwYcMQGRkpv+6++27ExsY2OSB86tSpCAgIwEMPPYRffvkFeXl5SEtLw9y5c29psP6tnve8efOwceNG5OXlISsrC1u2bEGvXr1uuM+vv/6KN998E0ePHsUHH3yA//73v/L7jBw5ErGxsZg4cSI2btyIU6dOYefOnfh//+//Ye/evVaf1+3o0aMHpk6diieeeAJr1qxBXl4e9uzZgzfeeOOGdwgSNSeGJiIH8+qrrza4FNerVy98+OGH+OCDD9CvXz/s3r270TvLbtXSpUvxxhtvoF+/fvjll1/w3XffISAgAAAQGhqKX3/9FUajEaNGjUJkZCTmzp0LjUZjNn7KEnPmzEFiYiISExMRFRWFlJQUfP/99+jevbtVx+natSuys7MxcuRILFy4EP369cPAgQPx/vvvY8GCBXj11VdRXV2Nr776Co888kijx3jkkUfw1VdfNTq/koeHB7Zv347w8HA8/PDD6NWrF2bMmIHKyspb6nm61fM2Go2YOXMmevXqhdGjR6Nnz5748MMPb7hPYmIiMjMzER0djVdffRVvv/02Ro0aBaDu8tiGDRswbNgwzJgxAz169MBjjz2GU6dOISgoyOrzul0rV67EE088gcTERPTs2RMTJkzArl27EBYW1uK1EAGAJK7/15eIiFqlTp06Yd68eZg3b57SpRA5JPY0EREREVmAoYmIiIjIArw8R0RERGQB9jQRERERWYChiYiIiMgCDE1EREREFmBoIiIiIrIAQxMRERGRBRiaiIiIiCzA0ERERERkAYYmIiIiIgv8f5/KnsaLi1iEAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "counts = df_final['g_anno_error_cnt'].value_counts()\n",
    "counts.plot()\n",
    "plt.xlabel(\"Number of GA errors per line\")\n",
    "plt.ylabel(\"Number of Lines with Errors\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 406
    },
    "id": "siwWn3hElDqb",
    "outputId": "539e4a03-a3a3-4a73-f544-89902d4fc345"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAGwCAYAAAC0HlECAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABOoElEQVR4nO3deXxTZaI+8OekTdM9pXsrpWwtAi0gVKEVCiqrLHJ1RhRuhSuizrCIFBfwN4KCgDKiM6KOOg44M2odRdBBraBAS2UvVFkKbaHQAi3d0qQtXZP39wc0EEohKUlPkzzfzyefS855kzwJDnnuOW/eIwkhBIiIiIjohhRyByAiIiKyByxNRERERGZgaSIiIiIyA0sTERERkRlYmoiIiIjMwNJEREREZAaWJiIiIiIzuModwJEYDAacP38ePj4+kCRJ7jhERERkBiEEqqqqEB4eDoWi9eNJLE1WdP78eURERMgdg4iIiNqgsLAQnTt3bnU/S5MV+fj4ALj0ofv6+sqchoiIiMyh0+kQERFh/B5vDUuTFTWfkvP19WVpIiIisjM3m1rDieBEREREZmBpIiIiIjIDSxMRERGRGViaiIiIiMzA0kRERERkBpYmIiIiIjOwNBERERGZgaWJiIiIyAwsTURERERmYGkiIiIiMgNLExEREZEZWJqIiIiIzMDSZAcamgzYf7oCQgi5oxARETktlqYOrklvQPzKn/H7v+3GydJqueMQERE5LZamDs7VRYE+4b4AgLScMpnTEBEROS+WJjswPDoIAJCeUypzEiIiIufF0mQHEi+Xpr355ahr1MuchoiIyDmxNNmBqGBvhPq6o67x0oRwIiIian8sTXZAkiQkRgcC4Ck6IiIiubA02YlE47wmTgYnIiKSA0uTnRjaMxAKCThxoQrF2jq54xARETkdliY74efphn6d/QAA6bk8RUdERNTeWJrsSCKXHiAiIpINS5MdGX55MnhGXhn0Bl5ShYiIqD2xNNmR/p394OPuisqLjTh8Tit3HCIiIqfC0mRHXF0UGNqTSw8QERHJgaXJznBeExERkTxYmuxMc2k6VFgJXV2jzGmIiIicB0uTnbnNzwM9grygNwjsyuNCl0RERO2FpckONR9tSuPq4ERERO2GpckOXT2vSQguPUBERNQeWJrs0JBuAXBzVeBcZS1OldXIHYeIiMgpsDTZIQ83F9zV1R8Af0VHRETUXlia7FRiNNdrIiIiak8sTXaqeV7TnlMVqG/Sy5yGiIjI8bE02aleIT4I8VWhtlGPA6c1cschIiJyeCxNdkqSJAyL4urgRERE7YWlyY5dWa+JpYmIiMjWWJrs2NCegZAk4HhxFUp0dXLHISIicmgsTXbM38sNsbepAQDpuVwdnIiIyJZYmuxcIuc1ERERtQuWJjvXPK8pI68MBgMvqUJERGQrLE127o4ufvBWuaKipgFHzmvljkNEROSwZC1NK1euxJ133gkfHx8EBwdj8uTJOHHihMkYIQSWLl2K8PBweHh4YMSIETh69KjJmPr6esydOxeBgYHw8vLCpEmTcPbsWZMxGo0GSUlJUKvVUKvVSEpKQmVlpcmYgoICTJw4EV5eXggMDMS8efPQ0NBgk/duLUoXBRJ6BADgKToiIiJbkrU0paWlYfbs2dizZw+2bt2KpqYmjB49GjU1Vy5C+8Ybb2DNmjVYu3Yt9u/fj9DQUIwaNQpVVVXGMfPnz8fGjRuRkpKCjIwMVFdXY8KECdDrr6yUPXXqVGRlZSE1NRWpqanIyspCUlKScb9er8f48eNRU1ODjIwMpKSkYMOGDUhOTm6fD+MWNJ+iS8/hZHAiIiKbER1ISUmJACDS0tKEEEIYDAYRGhoqVq1aZRxTV1cn1Gq1+Nvf/iaEEKKyslIolUqRkpJiHHPu3DmhUChEamqqEEKIY8eOCQBiz549xjG7d+8WAMTx48eFEEJ8//33QqFQiHPnzhnHfP7550KlUgmtVmtWfq1WKwCYPd5aCsprROQLm0WPRd8JXW1Du742ERGRvTP3+7tDzWnSai/NyfH39wcA5Ofno7i4GKNHjzaOUalUGD58OHbt2gUAyMzMRGNjo8mY8PBwxMTEGMfs3r0barUagwcPNo4ZMmQI1Gq1yZiYmBiEh4cbx4wZMwb19fXIzMy8bt76+nrodDqTmxwi/D3RLdALTQaBXSfLZclARETk6DpMaRJCYMGCBRg6dChiYmIAAMXFxQCAkJAQk7EhISHGfcXFxXBzc0OnTp1uOCY4OLjFawYHB5uMufZ1OnXqBDc3N+OYa61cudI4R0qtViMiIsLSt201iVGBADiviYiIyFY6TGmaM2cOfvvtN3z++ect9kmSZHJfCNFi27WuHXO98W0Zc7VFixZBq9Uab4WFhTfMZEvGeU25pRCCSw8QERFZW4coTXPnzsW3336L7du3o3PnzsbtoaGhANDiSE9JSYnxqFBoaCgaGhqg0WhuOObChQstXre0tNRkzLWvo9Fo0NjY2OIIVDOVSgVfX1+Tm1yGdA+A0kVCYUUtTpdflC0HERGRo5K1NAkhMGfOHHz99dfYtm0bunXrZrK/W7duCA0NxdatW43bGhoakJaWhoSEBADAoEGDoFQqTcYUFRXhyJEjxjHx8fHQarXYt2+fcczevXuh1WpNxhw5cgRFRUXGMVu2bIFKpcKgQYOs/+atzEvlirjIS3PBeIqOiIjI+mQtTbNnz8a///1vfPbZZ/Dx8UFxcTGKi4tRW1sL4NLpsvnz52PFihXYuHEjjhw5ghkzZsDT0xNTp04FAKjVasycORPJycn4+eefcejQIfzv//4vYmNjMXLkSABA7969MXbsWMyaNQt79uzBnj17MGvWLEyYMAG9evUCAIwePRp9+vRBUlISDh06hJ9//hkLFy7ErFmzZD2CZIkrSw+wNBEREVmdrX/GdyMArntbt26dcYzBYBBLliwRoaGhQqVSicTERHH48GGT56mtrRVz5swR/v7+wsPDQ0yYMEEUFBSYjCkvLxfTpk0TPj4+wsfHR0ybNk1oNBqTMWfOnBHjx48XHh4ewt/fX8yZM0fU1dWZ/X7kWnKg2ZFzlSLyhc2i959+EPWNelkyEBER2Rtzv78lIThr2Fp0Oh3UajW0Wq0sR6cMBoG7VvyMsup6fDZrMBJ6BLZ7BiIiIntj7vd3h5gITtahUEhXLT3A1cGJiIisiaXJwXBeExERkW2wNDmYoZePNB0r0qG0ql7mNERERI6DpcnBBHqrEHPbpfOxO3N5tImIiMhaWJocUGIUT9ERERFZG0uTA2qe17QztwwGA38cSUREZA0sTQ5oYJdO8HJzQXlNA44V6eSOQ0RE5BBYmhyQm6sC8ZfXaErjKToiIiKrYGlyUMOjm9drYmkiIiKyBpYmB9U8rynzjAbV9U0ypyEiIrJ/LE0OKjLAC5EBnmgyCOw+WS53HCIiIrvH0uTAuPQAERGR9bA0OTDjJVW4yCUREdEtY2lyYPE9AuCqkHCm/CLOlNfIHYeIiMiusTQ5MG+VKwZFdgLAU3RERES3iqXJwTWfokvLKZM5CRERkX1jaXJwwy+Xpt0ny9DQZJA5DRERkf1iaXJwfcJ8EeDlhpoGPQ4WaOSOQ0REZLdYmhycQiFhWBRXByciIrpVLE1OgEsPEBER3TqWJicw7PIil0fO6VBWXS9zGiIiIvvE0uQEgnxU6BPmCwDIyOWv6IiIiNqCpclJGE/RcV4TERFRm7A0OYnE6MuTwXPLYDAImdMQERHZH5YmJxEX6Q9PNxeUVdcju1gndxwiIiK7w9LkJNxcFYjvHgAASOfq4ERERBZjaXIinNdERETUdixNTqS5NB04U4Ga+iaZ0xAREdkXliYn0jXAExH+HmjUC+w5VS53HCIiIrvC0uREJElCYhRP0REREbUFS5OTuXJJFU4GJyIisgRLk5NJ6BEAV4WE/LIaFFZclDsOERGR3WBpcjI+7koM7NIJAJDGU3RERERms7g0HTx4EIcPHzbe/+abbzB58mQsXrwYDQ0NVg1HtmFcHZyliYiIyGwWl6annnoKOTk5AIBTp07hkUcegaenJ7788ks8//zzVg9I1tc8r2nXyXI06g0ypyEiIrIPFpemnJwcDBgwAADw5ZdfIjExEZ999hnWr1+PDRs2WDsf2UBMuBr+Xm6orm/CoYJKueMQERHZBYtLkxACBsOloxM//fQT7r//fgBAREQEysr4iyx7oFBIGNqTp+iIiIgsYXFpiouLw/Lly/Gvf/0LaWlpGD9+PAAgPz8fISEhVg9ItnFl6QGWJiIiInNYXJrefvttHDx4EHPmzMFLL72Enj17AgC++uorJCQkWD0g2UZi1KUjTYfPaVFRwwn8REREN+NqyWC9Xg+NRoO0tDT4+/ub7Fu9ejVcXFysGo5sJ9jXHbeH+uB4cRV25pbigQG3yR2JiIioQ7PoSJOLiwvGjBkDrVbbYp+7uzuUSqXVgpHtDW8+RZfDuWhEREQ3Y/HpudjYWJw6dcoWWaidNc9r2plbCiGEzGmIiIg6NotL02uvvYaFCxdi8+bNKCoqgk6nM7mR/Yjr2gkeSheUVNXjeHGV3HGIiIg6NIvmNAHA2LFjAQCTJk2CJEnG7UIISJIEvV5vvXRkUypXFwzp7o/tJ0qRnlOK3mG+ckciIiLqsCwuTdu3b7dFDpJJYnTQpdKUW4qnhveQOw4REVGHZXFpGj58uC1ykEya5zXtz9fgYkMTPN0s/k+CiIjIKbTpG7KyshIff/wxsrOzIUkS+vTpg8cffxxqtdra+cjGugd64TY/D5yrrMXeUxW45/ZguSMRERF1SBZPBD9w4AB69OiBt956CxUVFSgrK8OaNWvQo0cPHDx40BYZyYYkSTIebUrjJVWIiIhaZXFpevbZZzFp0iScPn0aX3/9NTZu3Ij8/HxMmDAB8+fPt0FEsrXh0ZevQ8dLqhAREbWqTUeaXnjhBbi6Xjmz5+rqiueffx4HDhywajhqHwk9A+GikHCqtAZnNRfljkNERNQhWVyafH19UVBQ0GJ7YWEhfHx8rBKK2pevuxJ3RPgB4OrgRERErbG4NE2ZMgUzZ87EF198gcLCQpw9exYpKSl44okn8Oijj9oiI7WDROMlVXiKjoiI6Hos/vXcn//8Z0iShMceewxNTU0AAKVSiT/84Q9YtWqV1QNS+0iMDsKarTn45WQZmvQGuLpY3KeJiIgcmkWlSa/XY/fu3ViyZAlWrlyJkydPQgiBnj17wtPT01YZqR3E3qaGn6cSlRcbkVVYibiu/nJHIiIi6lAsOpzg4uKCMWPGQKvVwtPTE7GxsejXrx8LkwNwUUgY2vPyr+h4io6IiKgFi8/BxMbG4tSpU7bIQjIzrteUy8ngRERE17K4NL322mtYuHAhNm/ejKKiIuh0OpMb2a/EqEul6bezldDUNMichoiIqGOxeCL42LFjAQCTJk2CJEnG7UIISJIEvV5vvXTUrkLV7ugV4oMTF6qQkVeGif3D5Y5ERETUYVhcmrZv326LHNRBJEYH4sSFKqTnlLI0ERERXcWi0tTY2IilS5figw8+QHR0tK0ykYwSo4Pw0c58pOeWGo8eEhERkYVzmpRKJY4cOcIvUgd2Z1d/uCsVuKCrR86FarnjEBERdRgWTwR/7LHH8PHHH9siC3UA7koXDO4WAIBLDxAREV3N4jlNDQ0N+Pvf/46tW7ciLi4OXl5eJvvXrFljtXAkj8ToIKTllCI9txSzErvLHYeIiKhDsLg0HTlyBAMHDgQA5OTkmOzjaTvHMDw6EMsA7M2vQG2DHh5uLnJHIiIikh1/PUct9AjyRrjaHee1ddibX44RvYLljkRERCQ7q16VtaSkxKLx6enpmDhxIsLDwyFJEjZt2mSyf8aMGZAkyeQ2ZMgQkzH19fWYO3cuAgMD4eXlhUmTJuHs2bMmYzQaDZKSkqBWq6FWq5GUlITKykqTMQUFBZg4cSK8vLwQGBiIefPmoaHBORd4lCTJuDp4eg5XByciIgIsKE2enp4oLb0yMXjs2LEoKioy3r9w4QLCwsIsevGamhr0798fa9eubXVM8+s0377//nuT/fPnz8fGjRuRkpKCjIwMVFdXY8KECSaLbE6dOhVZWVlITU1FamoqsrKykJSUZNyv1+sxfvx41NTUICMjAykpKdiwYQOSk5Mtej+OxFiacjkZnIiICLDg9FxdXR2EEMb7v/zyC2pra03GXL3fHOPGjcO4ceNuOEalUiE0NPS6+7RaLT7++GP861//wsiRIwEA//73vxEREYGffvoJY8aMQXZ2NlJTU7Fnzx4MHjwYAPDRRx8hPj4eJ06cQK9evbBlyxYcO3YMhYWFCA+/tKDjm2++iRkzZuC1116Dr6/vdV+/vr4e9fX1xvuOdBmZu3sEQiEBeSXVOF9Zi3A/D7kjERERycqqp+dsMRF8x44dCA4ORnR0NGbNmmVyCjAzMxONjY0YPXq0cVt4eDhiYmKwa9cuAMDu3buhVquNhQkAhgwZArVabTImJibGWJgAYMyYMaivr0dmZmar2VauXGk85adWqxEREWG19y03tacSAyL8AHDpASIiIsDKpcnaxo0bh08//RTbtm3Dm2++if379+Pee+81Ht0pLi6Gm5sbOnXqZPK4kJAQFBcXG8cEB7ecyBwcHGwyJiQkxGR/p06d4ObmZhxzPYsWLYJWqzXeCgsLb+n9djQ8RUdERHSF2afnmidit3bfFqZMmWL8c0xMDOLi4hAZGYnvvvsODz74YKuPu/byH9fL2ZYx11KpVFCpVDd9H/YqMToIb/+Ui4zcMjTpDXB16dAdm4iIyKbM/hYUQiA6Ohr+/v7w9/dHdXU17rjjDuP922+/3ZY5AQBhYWGIjIxEbm4uACA0NBQNDQ3QaDQm40pKSoxHjkJDQ3HhwoUWz1VaWmoy5tojShqNBo2NjS2OQDmT/p39oPZQQlfXhF/PauWOQ0REJCuzjzStW7fOljnMUl5ejsLCQuOv9AYNGgSlUomtW7fi4YcfBgAUFRXhyJEjeOONNwAA8fHx0Gq12LdvH+666y4AwN69e6HVapGQkGAc89prr6GoqMj43Fu2bIFKpcKgQYPa+212GC4KCUN7BuK7w0VIzynFoMhON38QERGRg5KEpT95s6Lq6mrk5eUBAO644w6sWbMG99xzj/Ho1dKlS/HQQw8hLCwMp0+fxuLFi1FQUIDs7Gz4+PgAAP7whz9g8+bNWL9+Pfz9/bFw4UKUl5cjMzMTLi6XVrIeN24czp8/jw8++AAA8OSTTyIyMhL//e9/AVxacmDAgAEICQnB6tWrUVFRgRkzZmDy5Ml45513zH4/Op0OarUaWq221V/c2Zsv9hfghQ2HcUcXP2z8491yxyEiIrI6s7+/hYy2b98uALS4TZ8+XVy8eFGMHj1aBAUFCaVSKbp06SKmT58uCgoKTJ6jtrZWzJkzR/j7+wsPDw8xYcKEFmPKy8vFtGnThI+Pj/Dx8RHTpk0TGo3GZMyZM2fE+PHjhYeHh/D39xdz5swRdXV1Fr0frVYrAAitVtumz6MjOl95UUS+sFl0e3GzqKxpkDsOERGR1Zn7/S3rkSZH44hHmgBg1Jo05JZU492pAzG+n2ULmBIREXV05n5/8+dQdFNXLqnCpQeIiMh5sTTRTV29XhMPTBIRkbNiaaKbGtzNHypXBYq0dcgrqZY7DhERkSzMXnKgmV6vx/r16/Hzzz+jpKQEBoPBZP+2bdusFo46BnelC+7q5o+duWVIyylFVIiP3JGIiIjancWl6ZlnnsH69esxfvx4xMTE2HxVcOoYhkcHYWduGdJzy/DEsO5yxyEiImp3FpemlJQU/Oc//8H9999vizzUQSVGBwHfZWPvqXLUNerhrnSROxIREVG7snhOk5ubG3r27GmLLNSBRQV7I9TXHfVNBuzLr5A7DhERUbuzuDQlJyfjL3/5C39F5WQkSUJidCAALj1ARETOyazTcw8++KDJ/W3btuGHH35A3759oVQqTfZ9/fXX1ktHHUpidBD+c+As0nNZmoiIyPmYVZrUarXJ/f/5n/+xSRjq2Ib2DIRCAnIuVKNIW4swtYfckYiIiNqNWaVp3bp1ts5BdsDP0w39Ovshq7ASO3PK8PCdEXJHIiIiajcWz2m69957UVlZ2WK7TqfDvffea41M1IE1rw6exlN0RETkZCwuTTt27EBDQ0OL7XV1ddi5c6dVQlHHNfzyZPCM3DLoDfwxABEROQ+z12n67bffjH8+duwYiouLjff1ej1SU1Nx2223WTcddTj9O/vBx90V2tpG/Ha2End06SR3JCIionZhdmkaMGAAJEmCJEnXPQ3n4eGBd955x6rhqONxdVFgaM9A/HCkGOk5ZSxNRETkNMwuTfn5+RBCoHv37ti3bx+CgoKM+9zc3BAcHAwXF64S7QwSo4MulabcUjwzMkruOERERO3C7NIUGRkJAC0u0EvOp3kyeFZhJbS1jVB7KG/yCCIiIvtnVmn69ttvMW7cOCiVSnz77bc3HDtp0iSrBKOO6zY/D/QI8sLJ0hrsyivDuNgwuSMRERHZnFmlafLkySguLkZwcDAmT57c6jhJkqDX662VjTqwxOggnCytQXpuKUsTERE5BbOWHDAYDAgODjb+ubUbC5PzaD5Fl55TxusQEhGRU7B4naaLFy/aIgfZmSHdAuDmqsC5ylqcLK2ROw4REZHNWVya/Pz8kJCQgMWLF+PHH39ETQ2/MJ2Rh5sL7urqDwBIz+Hq4ERE5PgsLk1paWmYNGkSDh48iN///vfo1KkThgwZghdffBE//PCDLTJSB5V4eXXwdF5ShYiInIAkbmFCil6vx/79+/G3v/0Nn376qdPPa9LpdFCr1dBqtfD19ZU7js0dL9Zh7Ns74a5UIOvl0XBXcp0uIiKyP+Z+f5u9TtPVjh8/jh07diAtLQ07duxAY2MjJk6ciOHDh7c5MNmfXiE+CPFV4YKuHgdOazA0KlDuSERERDZjcWkKDQ1FY2Mj7r33XowYMQKLFy9GbGysLbJRBydJEoZFBeGrzLNIzy1laSIiIodm8Zym0NBQVFdXo6CgAAUFBTh79iyqq6ttkY3swJWlBziviYiIHJvFpSkrKwsXLlzASy+9hKamJvzpT39CUFAQBg8ejBdffNEWGakDG9YzEJIEHC+uwgVdndxxiIiIbOaWJoJXVFRgx44d+Oabb/DZZ59xIriTTQRv9sDaDPx6VovVv+uH38dFyB2HiIjIIuZ+f1t8pGnjxo145pln0L9/fwQHB+MPf/gDampq8NZbb+G33367pdBkn4yn6HLLZE5CRERkOxZPBH/qqaeQmJiIWbNmYcSIEYiJibFFLrIjidFBeGdbHjJyS6E3CLgoJLkjERERWZ3FpamkpMQWOciODYjwg4/KFZqLjThyTov+EX5yRyIiIrI6i0/PEV1L6aJAQs8AAPwVHREROS6WJrKKK/OaWJqIiMgxsTSRVSRGXSpNBwsqoatrlDkNERGR9bE0kVVE+Huie6AX9AaBXXnlcschIiKyulsuTTqdDps2bUJ2drY18pAd4yk6IiJyZBaXpocffhhr164FANTW1iIuLg4PP/ww+vXrhw0bNlg9INmPxOhL155LzynFLayZSkRE1CFZXJrS09MxbNgwAJcWuhRCoLKyEn/961+xfPlyqwck+zGkewDcXBQ4q6lFflmN3HGIiIisyuLSpNVq4e/vDwBITU3FQw89BE9PT4wfPx65ublWD0j2w9PNFXFdOwHg0gNEROR4LC5NERER2L17N2pqapCamorRo0cDADQaDdzd3a0ekOwLL6lCRESOyuLSNH/+fEybNg2dO3dGWFgYRowYAeDSabvY2Fhr5yM707z0wO6T5ahvct6LNxMRkeOx+DIqf/zjH3HXXXehsLAQo0aNgkJxqXd1796dc5oIvcN8EOSjQmlVPTJPa5DQM1DuSERERFbRpiUH4uLiMH78eJw7dw5NTU0AgPHjx+Puu++2ajiyP5IkYVjUpaKUxqUHiIjIgVhcmi5evIiZM2fC09MTffv2RUFBAQBg3rx5WLVqldUDkv0Z3jyvKYfzmoiIyHFYXJoWLVqEX3/9FTt27DCZ+D1y5Eh88cUXVg1H9mloz0BIEpBdpENJVZ3ccYiIiKzC4tK0adMmrF27FkOHDoUkScbtffr0wcmTJ60ajuxTgLcKMeFqAMBOHm0iIiIHYXFpKi0tRXBwcIvtNTU1JiWKnJtxdXDOayIiIgdhcWm688478d133xnvNxeljz76CPHx8dZLRnateemBnbllMBh4SRUiIrJ/Fi85sHLlSowdOxbHjh1DU1MT/vKXv+Do0aPYvXs30tLSbJGR7NDAyE7wVrmioqYBR8/rENtZLXckIiKiW2LxkaaEhAT88ssvuHjxInr06IEtW7YgJCQEu3fvxqBBg2yRkeyQ0kWB+B4BAHiKjoiIHIPFR5oAIDY2Fp988om1s5CDSYwOwtZjF5CWU4rZ9/SUOw4REdEtaVNpMhgMyMvLQ0lJCQwGg8m+xMREqwQj+zf88rymg2c0qKprhI+7UuZEREREbWdxadqzZw+mTp2KM2fOQAjTCb6SJEGv5/XG6JIuAZ7oGuCJ0+UXsftkOUb3DZU7EhERUZtZPKfp6aefRlxcHI4cOYKKigpoNBrjraKiwhYZyY4lNq8OznlNRERk5yw+0pSbm4uvvvoKPXtyjgrdXGJUEP65+wwvqUJERHbP4iNNgwcPRl5eni2ykAOK7xEApYuEgoqLOF1WI3ccIiKiNrP4SNPcuXORnJyM4uJixMbGQqk0ndzbr18/q4Uj++elcsWgyE7Yc6oC6bml6BroJXckIiKiNrG4ND300EMAgMcff9y4TZIkCCE4EZyuKzE66FJpyinFY/Fd5Y5DRETUJhaXpvz8fFvkIAeWGBWEN1JPYPfJcjQ0GeDmavFZYSIiItlZXJoiIyNtkYMcWJ8wXwR6u6GsugGZZzTGlcKJiIjsiVml6dtvv8W4ceOgVCrx7bff3nDspEmTrBKMHIdCIWFYVBA2HjqH9NxSliYiIrJLkrh2hcrrUCgUKC4uRnBwMBSK1k+tOPucJp1OB7VaDa1WC19fX7njdCgbD53Fs1/8ir7hvvhu3jC54xARERmZ+/1t1pGmqy+Vcu1lU4jMMezyJVWOntehtKoeQT4qmRMRERFZxmozcgsLC01+UUd0tUBvFfqGX2rvGXlcHZyIiOyP1UpTRUUFPvnkE4sek56ejokTJyI8PBySJGHTpk0m+4UQWLp0KcLDw+Hh4YERI0bg6NGjJmPq6+sxd+5cBAYGwsvLC5MmTcLZs2dNxmg0GiQlJUGtVkOtViMpKQmVlZUmYwoKCjBx4kR4eXkhMDAQ8+bNQ0NDg0Xvh27MeEkVrg5ORER2SNbfftfU1KB///5Yu3btdfe/8cYbWLNmDdauXYv9+/cjNDQUo0aNQlVVlXHM/PnzsXHjRqSkpCAjIwPV1dWYMGGCydyqqVOnIisrC6mpqUhNTUVWVhaSkpKM+/V6PcaPH4+amhpkZGQgJSUFGzZsQHJysu3evBNKvHyKbmduKQyGm06lIyIi6liElWRlZQmFQtHmxwMQGzduNN43GAwiNDRUrFq1yritrq5OqNVq8be//U0IIURlZaVQKpUiJSXFOObcuXNCoVCI1NRUIYQQx44dEwDEnj17jGN2794tAIjjx48LIYT4/vvvhUKhEOfOnTOO+fzzz4VKpRJarbbVzHV1dUKr1RpvhYWFAsANH+PM6hv1os+ffhCRL2wWh89Wyh2HiIhICCGEVqs16/u7w64ymJ+fj+LiYowePdq4TaVSYfjw4di1axcAIDMzE42NjSZjwsPDERMTYxyze/duqNVqDB482DhmyJAhUKvVJmNiYmIQHh5uHDNmzBjU19cjMzOz1YwrV640nvJTq9WIiIiwzpt3UG6uCuNyA+m5nNdERET2xezFLR988MEb7r92jtCtKi4uBgCEhISYbA8JCcGZM2eMY9zc3NCpU6cWY5of37xUwrWCg4NNxlz7Op06dYKbm5txzPUsWrQICxYsMN7X6XQsTjeRGB2En7JLkJ5Tij+O6Cl3HCIiIrOZXZrUavVN9z/22GO3HOhakiSZ3BeXr3F3I9eOud74toy5lkqlgkrFn85bonleU+YZDWrqm+ClsnhReiIiIlmY/Y21bt06W+ZoITQ0FMClo0BhYWHG7SUlJcajQqGhoWhoaIBGozE52lRSUoKEhATjmAsXLrR4/tLSUpPn2bt3r8l+jUaDxsbGFkeg6NZ0DfRCF39PFFRcxO6T5RjZh58vERHZhw47p6lbt24IDQ3F1q1bjdsaGhqQlpZmLESDBg2CUqk0GVNUVIQjR44Yx8THx0Or1WLfvn3GMXv37oVWqzUZc+TIERQVFRnHbNmyBSqVCoMGDbLp+3RGidGBADiviYiI7Ius50aqq6uRl5dnvJ+fn4+srCz4+/ujS5cumD9/PlasWIGoqChERUVhxYoV8PT0xNSpUwFcOiU4c+ZMJCcnIyAgAP7+/li4cCFiY2MxcuRIAEDv3r0xduxYzJo1Cx988AEA4Mknn8SECRPQq1cvAMDo0aPRp08fJCUlYfXq1aioqMDChQsxa9YsXg7FBhKjgvDvPQVIz2FpIiIiO9IOv+Rr1fbt2wWAFrfp06cLIS4tO7BkyRIRGhoqVCqVSExMFIcPHzZ5jtraWjFnzhzh7+8vPDw8xIQJE0RBQYHJmPLycjFt2jTh4+MjfHx8xLRp04RGozEZc+bMGTF+/Hjh4eEh/P39xZw5c0RdXZ1F78fcnyw6O11tg+ix6DsR+cJmcaasRu44RETk5Mz9/jbrgr1kHl6w13wPf7Ab+/IrsGxyDJKGRModh4iInJi5399mzWkaOHAgNBoNAODVV1/FxYsXrZOSnNZw4yVVeIqOiIjsg1mlKTs7GzU1NQCAV155BdXV1TYNRY6veemB3SfL0ag3yJyGiIjo5syaCD5gwAD83//9H4YOHQohBP785z/D29v7umNffvllqwYkx9Q33BcBXm4or2nAwTMaDO4eIHckIiKiGzKrNK1fvx5LlizB5s2bIUkSfvjhB7i6tnyoJEksTWQWhULC0KhAfJN1Hum5pSxNRETU4ZlVmnr16oWUlBQAgEKhwM8//3zdS5MQWSIxKuhSacopw3Nj5E5DRER0Yxav02QwcP4JWcewy4tcHjmvRXl1PQK8eUkaIiLquNq0IvjJkycxd+5cjBw5EqNGjcK8efNw8uRJa2cjBxfs447eYb4QAsjIK5M7DhER0Q1ZXJp+/PFH9OnTB/v27UO/fv0QExODvXv3om/fviaXMyEyR/MlVdK49AAREXVwFp+ee/HFF/Hss89i1apVLba/8MILGDVqlNXCkeMbHhWED9JOYWduGYQQkCRJ7khERETXZfGRpuzsbMycObPF9scffxzHjh2zSihyHoO6doKH0gWlVfXILqqSOw4REVGrLC5NQUFByMrKarE9KyuLv6gji6lcXRDf49JyA+m5PEVHREQdl8Wn52bNmoUnn3wSp06dQkJCAiRJQkZGBl5//XUkJyfbIiM5uMSoQGw7XoL0nFI8PbyH3HGIiIiuy+LS9Kc//Qk+Pj548803sWjRIgBAeHg4li5dinnz5lk9IDm+xMvXoTtwWoOLDU3wdLP4P0siIiKbk4QQoq0Prqq6NAfFx8fHaoHsmblXSSZTQggMe2M7zmpq8Y8Zcbj39hC5IxERkRMx9/u7Tes0NfPx8WFholsmSZLxaFN6DtdrIiKijumWShORtSRGNZcmTgYnIqKOiaWJOoSEngFwUUg4VVaDwoqLcschIiJqgaWJOgRfdyUGdvEDwKUHiIioY7KoNDU2NuKee+5BTk6OrfKQExvGU3RERNSBWVSalEoljhw5wktdkE00TwbflVeORr1B5jRERESmLD4999hjj+Hjjz+2RRZycrG3qeHnqURVfROyCivljkNERGTC4lUEGxoa8Pe//x1bt25FXFwcvLy8TPavWbPGauHIubgoJAztGYjNvxUhPacUd3b1lzsSERGRkcWl6ciRIxg4cCAAtJjbxNN2dKsSo4OMpSl5dC+54xARERlZXJq2b99uixxEAK6s1/TbOS0qahrg7+UmcyIiIqJL2rzkQF5eHn788UfU1tYCuHQpDKJbFap2R68QHwgBZORxdXAiIuo4LC5N5eXluO+++xAdHY37778fRUVFAIAnnngCycnJVg9IzicxOhAAlx4gIqKOxeLS9Oyzz0KpVKKgoACenp7G7VOmTEFqaqpVw5Fzal56YGduKY9gEhFRh2HxnKYtW7bgxx9/ROfOnU22R0VF4cyZM1YLRs7rzq7+cFcqcEFXjxMXqnB7aOtXnCYiImovFh9pqqmpMTnC1KysrAwqlcoqoci5uStdMLhbAACeoiMioo7D4tKUmJiIf/7zn8b7kiTBYDBg9erVuOeee6wajpxX8ym69BxOBicioo7B4tNzq1evxogRI3DgwAE0NDTg+eefx9GjR1FRUYFffvnFFhnJCQ2PDsQyAPtOV6C2QQ8PNxe5IxERkZOz+EhTnz598Ntvv+Guu+7CqFGjUFNTgwcffBCHDh1Cjx49bJGRnFCPIG+Eq93R0GTAnvxyueMQERFZfqQJAEJDQ/HKK69YOwuRkSRJSIwOQsr+QqTnlOKeXsFyRyIiIifXptKk0Wjw8ccfIzs7G5IkoXfv3vi///s/+PvzWmFkPVeXJiIiIrlZfHouLS0N3bp1w1//+ldoNBpUVFTgr3/9K7p164a0tDRbZCQndXePQCgk4GRpDc5V1sodh4iInJzFpWn27Nl4+OGHkZ+fj6+//hpff/01Tp06hUceeQSzZ8+2RUZyUmpPJQZE+AHg0gNERCQ/i0vTyZMnkZycDBeXK79mcnFxwYIFC3Dy5EmrhiO6svQASxMREcnL4tI0cOBAZGdnt9ienZ2NAQMGWCMTkVFzacrIK0OT3iBzGiIicmZmTQT/7bffjH+eN28ennnmGeTl5WHIkCEAgD179uDdd9/FqlWrbJOSnFb/zn5QeyihrW3Er2crMSiSPzYgIiJ5SMKMK6IqFApIknTTi6dKkgS9Xm+1cPZGp9NBrVZDq9XC15fXS7OW2Z8exHeHizDvvigsGBUtdxwiInIw5n5/m3WkKT8/32rBiCyVGB2I7w4XIT2nlKWJiIhkY1ZpioyMtHUOolY1z2v67WwlKi82wM/TTeZERETkjNq0uOW5c+fwyy+/oKSkBAaD6eTcefPmWSUYUbMwtQeigr2RW1KNjLwyTOgXLnckIiJyQhaXpnXr1uHpp5+Gm5sbAgICIEmScZ8kSSxNZBOJ0UHILalGek4pSxMREcnC4iUHXn75Zbz88svQarU4ffo08vPzjbdTp07ZIiPRVes1ld30BwlERES2YHFpunjxIh555BEoFBY/lKjNBnfzh8pVgWJdHXJLquWOQ0RETsji5jNz5kx8+eWXtshC1Cp3pQvu6nZpjSauDk5ERHKweE7TypUrMWHCBKSmpiI2NhZKpdJk/5o1a6wWjuhqw6ODsDO3DGk5pXhiWHe54xARkZOxuDStWLECP/74I3r16gUALSaCE9lKYnQQ8F029uVXoK5RD3ely80fREREZCUWl6Y1a9bgH//4B2bMmGGDOEStiwr2RqivO4p1ddibX4HhlyeHExERtQeL5zSpVCrcfffdtshCdEOSJCExOhAA5zUREVH7s7g0PfPMM3jnnXdskYXopq4sPcDSRERE7cvi03P79u3Dtm3bsHnzZvTt27fFRPCvv/7aauGIrjW0ZyAUEpBbUo3zlbUI9/OQOxIRETkJi0uTn58fHnzwQVtkIbopP0839Ovsh6zCSuzMLcWUO7vIHYmIiJxEmy6jQiSnxOggZBVWIj2njKWJiIjaDZf1Jrsz/PJk8Iy8MugNvKQKERG1D4uPNHXr1u2G6zHx+nNka/07+8HH3RXa2kb8erYSA7t0kjsSERE5AYtL0/z5803uNzY24tChQ0hNTcVzzz1nrVxErXJ1UWBoz0D8cKQY6TmlLE1ERNQuLC5NzzzzzHW3v/vuuzhw4MAtByIyR2J0kLE0zR8ZLXccIiJyAlab0zRu3Dhs2LDBWk9HdEPN6zVlFVZCe7FR5jREROQMrFaavvrqK/j7+1vr6Yhu6DY/D/QI8oJBAL+cLJM7DhEROQGLT8/dcccdJhPBhRAoLi5GaWkp3nvvPauGI7qRxOggnCytQXpOKe6PDZM7DhEROTiLS9PkyZNN7isUCgQFBWHEiBG4/fbbrZWL6KYSo4Ow7pfTSM8phRDihr/qJCIiulUWl6YlS5bYIgeRxYZ0C4CbqwLntXU4WVqNnsE+ckciIiIH1qEXt1y6dCkkSTK5hYaGGvcLIbB06VKEh4fDw8MDI0aMwNGjR02eo76+HnPnzkVgYCC8vLwwadIknD171mSMRqNBUlIS1Go11Go1kpKSUFlZ2R5vkW6Bh5sL7up6aR5dWg7nNRERkW2ZXZoUCgVcXFxueHN1tfjA1U317dsXRUVFxtvhw4eN+9544w2sWbMGa9euxf79+xEaGopRo0ahqqrKOGb+/PnYuHEjUlJSkJGRgerqakyYMAF6vd44ZurUqcjKykJqaipSU1ORlZWFpKQkq78Xsr7Ey6uDp+eUypyEiIgcndktZ+PGja3u27VrF9555x0IYf1LWri6upocXWomhMDbb7+Nl156yXgB4U8++QQhISH47LPP8NRTT0Gr1eLjjz/Gv/71L4wcORIA8O9//xsRERH46aefMGbMGGRnZyM1NRV79uzB4MGDAQAfffQR4uPjceLECfTq1cvq74msJzE6CCu+P469+eWoa9TDXekidyQiInJQZh9peuCBB1rcevXqhfXr1+PNN9/E73//e5w4ccLqAXNzcxEeHo5u3brhkUceMV6mJT8/H8XFxRg9erRxrEqlwvDhw7Fr1y4AQGZmJhobG03GhIeHIyYmxjhm9+7dUKvVxsIEAEOGDIFarTaOaU19fT10Op3JjdpXrxAfhPiqUNdowP7TFXLHISIiB9amOU3nz5/HrFmz0K9fPzQ1NSErKwuffPIJunSx7hXnBw8ejH/+85/48ccf8dFHH6G4uBgJCQkoLy9HcXExACAkJMTkMSEhIcZ9xcXFcHNzQ6dOnW44Jjg4uMVrBwcHG8e0ZuXKlcZ5UGq1GhEREW1+r9Q2kiRhWNSlhS55io6IiGzJotKk1WrxwgsvoGfPnjh69Ch+/vln/Pe//0VMTIxNwo0bNw4PPfQQYmNjMXLkSHz33XcALp2Ga3btz8zN+en5tWOuN96c51m0aBG0Wq3xVlhYeNP3RNbXvDp4OieDExGRDZldmt544w10794dmzdvxueff45du3Zh2LBhtszWgpeXF2JjY5Gbm2uc53Tt0aCSkhLj0afQ0FA0NDRAo9HccMyFCxdavFZpaWmLo1jXUqlU8PX1NblR+xvWMxCSBJy4UIVibZ3ccYiIyEGZXZpefPFF1NXVoWfPnvjkk0/w4IMPXvdmS/X19cjOzkZYWBi6deuG0NBQbN261bi/oaEBaWlpSEhIAAAMGjQISqXSZExRURGOHDliHBMfHw+tVot9+/YZx+zduxdardY4hjq2Tl5u6HebGgCQnstTdEREZBtm/3rusccea/cVlxcuXIiJEyeiS5cuKCkpwfLly6HT6TB9+nRIkoT58+djxYoViIqKQlRUFFasWAFPT09MnToVAKBWqzFz5kwkJycjICAA/v7+WLhwofF0HwD07t0bY8eOxaxZs/DBBx8AAJ588klMmDCBv5yzI4nRQfj1rBbpOaV4OI5zy4iIyPrMLk3r16+3YYzrO3v2LB599FGUlZUhKCgIQ4YMwZ49exAZGQkAeP7551FbW4s//vGP0Gg0GDx4MLZs2QIfnysrQ7/11ltwdXXFww8/jNraWtx3331Yv349XFyu/DT9008/xbx584y/sps0aRLWrl3bvm+WbklidBDe2ZaHjLwy6A0CLgpeUoWIiKxLErZYXMlJ6XQ6qNVqaLVazm9qZ416Awa+uhVV9U3YNPtuDIjwkzsSERHZCXO/vzv0ZVSIzKV0USChZwAALj1ARES2wdJEDuPK0gMsTUREZH0sTeQwEi8vcnmosBK6ukaZ0xARkaNhaSKHEeHvie6BXtAbBHblcaFLIiKyLpYmcijNp+jSuDo4ERFZGUsTOZTE6EAAl+Y18YehRERkTSxN5FCGdA+Am4sC5yprcaqsRu44RETkQFiayKF4urkirmsnAPwVHRERWRdLEzkcLj1ARES2wNJEDqd56YE9pypQ36SXOQ0RETkKliZyOL3DfBDko0Jtox4HTmvkjkNERA6CpYkcjiRJGBZ15Vd0RERE1sDSRA5puHG9JpYmIiKyDpYmckhDewZCkoDjxVUo0dXJHYeIiBwASxM5pABvFWLC1QCA9FyuDk5ERLeOpYkc1tWrgxMREd0qliZyWM1LD2TklcFg4CVViIjo1rA0kcMaGNkJ3ipXVNQ04Mh5rdxxiIjIzrE0kcNSuigQ3yMAAE/RERHRrWNpIod25ZIqnAxORES3hqWJHNrwy/OaDhZoUFXXKHMaIiKyZyxN5NC6BHiia4AnmgwCu06Wyx2HiIjsGEsTObwrp+g4r4mIiNqOpYkcXvPSA+m5pRCCSw8QEVHbsDSRw4vvEQCli4TCilqcLr8odxwiIrJTLE3k8LxUrhgU2QkAT9EREVHbsTSRU+C8JiIiulUsTeQUmuc17T5VjoYmg8xpiIjIHrE0kVPoE+aLQG83XGzQ48CZCrnjEBGRHWJpIqegUEgYFsXVwYmIqO1YmshpJEYHAuC8JiIiahuWJnIazUeajhXpUFpVL3MaIiKyNyxN5DQCvVXoG+4LANiZy6NNRERkGZYmcipceoCIiNqKpYmcSvPSAztzy2Aw8JIqRERkPpYmciqDIjvBy80F5TUNOFakkzsOERHZEZYmcipurgrE9wgAAKTxFB0REVmApYmcDuc1ERFRW7A0kdNpnteUeUaD6vommdMQEZG9YGkip9M10Atd/D3RZBDYfbJc7jhERGQnWJrIKXF1cCIishRLEzml5lN06VzkkoiIzMTSRE4pvkcAXBUSzpRfxJnyGrnjEBGRHWBpIqfk467EwMhOAHiKjoiIzMPSRE5r+OWlB9JyymROQkRE9oCliZxW87ym3SfL0NBkkDkNERF1dCxN5LT6hvsiwMsNNQ16HCzQyB2HiIg6OJYmcloKhYShUVx6gIiIzMPSRE6NSw8QEZG5WJrIqQ27vMjlkXM6lFXXy5yGiIg6MpYmcmrBPu7oHeYLAMjI5a/oiIiodSxN5PR4SRUiIjIHSxM5veHGeU1lMBiEzGmIiKijYmkipzeoayd4KF1QVl2P7GKd3HGIiKiDYmkip6dydUF8jwAAQDpXByciolawNBEBSOR6TUREdBMsTUQAEi9fh+7AmQrU1DfJnIaIiDoiV7kDEHUE3QK90LmTB85qajH+rzsR4e+JcLUHQtXuCPdzR6jaA+Fqd4Sq3eHjrpQ7LhERyYCliQiAJEn4nztuwzvb8nC6/CJOl19sdayPyhWhaneE+XkgzNedxYqIyElIQgj+xtpKdDod1Go1tFotfH195Y5DFhJC4HT5RZzT1KJIW4sibd3lWy2KtXU4X1kLXZ15p+6uLVZhfu4IU7sjTO2BMBYrIqIOxdzvbx5pIrpMkiR0C/RCt0CvVsfU1DehSFuH4stlqrViVVXfhKqSauSWVLf6XDcrVmF+HvBW8X+iREQdBf9FJrKAl8oVPYO90TPYu9UxVxer85fL1K0Uq7DLp/5YrIiI5MV/bYmsrO3FqvbKtquL1YVq5Fwwr1g1z6lisSIisj7+S0okg1stVkWVl/58K8Wq+deBLFZERObhv5JEHVRHKlbNvw5ksSIiZ8Z/Aa/x3nvvYfXq1SgqKkLfvn3x9ttvY9iwYXLHIrqudi9W7q6Xf/3HYkVEzof/ul3liy++wPz58/Hee+/h7rvvxgcffIBx48bh2LFj6NKli9zxiNrEkmJ19byq6xaruiZU1ZlXrK5eXoHFiogcAddpusrgwYMxcOBAvP/++8ZtvXv3xuTJk7Fy5cqbPp7rNJEju7ZYFVXWoVjXsliZw1vlCjdXBRTSpaUeFBKgkCQoJAkAoFBcuS8Z9+HyfQnSVWNMH48r+yXJ7DHS5de68hqX/2+LMS1zXcpy/ZwKCZBwedsNxzTvb5nzyp9bfy9Xj2nxXq7+4CXTv4er90rS9YdJkumDWh93/ReSWrzm9Z+79edqPWfruW6QuZVxN855/T2tfRbmuPZzveFYi5/bgrEWPLslz2spS547xNcdShfrXgWO6zRZqKGhAZmZmXjxxRdNto8ePRq7du267mPq6+tRX19vvK/T6WyakUhO5hyxqq5vanGUqlhXi/OVV45e6eqaUF3fBNS3+jRERK3aljwc3YNa/3fIlliaLisrK4Ner0dISIjJ9pCQEBQXF1/3MStXrsQrr7zSHvGI7IK3mcWqRFcHvUHAIACDEDAIAWH8My7fb95m/hghrn5OXL4vYDAAAjcYc1UW4NrXgMl+IYTxuW40xiAAgatf4zq5xDW5Whljkstw1Wte/TxXv6bhyr6rXX1XmGy/Zlwrd649LXH140yf75pxV+01yXCD8xytPXeL52jluW+c6dY/F9P3YdkJG4tGW3guyJLhluS29JSUJR+JsPDZLTlKZ20sTde49i9DCNHqX9CiRYuwYMEC432dToeIiAib5iOyd94qV3jL9P8lEhHdCpamywIDA+Hi4tLiqFJJSUmLo0/NVCoVVCpVe8QjIiIimVl3JpUdc3Nzw6BBg7B161aT7Vu3bkVCQoJMqYiIiKij4JGmqyxYsABJSUmIi4tDfHw8PvzwQxQUFODpp5+WOxoRERHJjKXpKlOmTEF5eTleffVVFBUVISYmBt9//z0iIyPljkZEREQy4zpNVsR1moiIiOyPud/fnNNEREREZAaWJiIiIiIzsDQRERERmYGliYiIiMgMLE1EREREZmBpIiIiIjIDSxMRERGRGViaiIiIiMzA0kRERERkBl5GxYqaF1fX6XQyJyEiIiJzNX9v3+wiKSxNVlRVVQUAiIiIkDkJERERWaqqqgpqtbrV/bz2nBUZDAacP38ePj4+kCTJas+r0+kQERGBwsJCXtPOhvg5tx9+1u2Dn3P74OfcPmz5OQshUFVVhfDwcCgUrc9c4pEmK1IoFOjcubPNnt/X15f/g2wH/JzbDz/r9sHPuX3wc24ftvqcb3SEqRknghMRERGZgaWJiIiIyAwsTXZApVJhyZIlUKlUckdxaPyc2w8/6/bBz7l98HNuHx3hc+ZEcCIiIiIz8EgTERERkRlYmoiIiIjMwNJEREREZAaWJiIiIiIzsDTZgffeew/dunWDu7s7Bg0ahJ07d8odyeGkp6dj4sSJCA8PhyRJ2LRpk9yRHM7KlStx5513wsfHB8HBwZg8eTJOnDghdyyH8/7776Nfv37GBQDj4+Pxww8/yB3L4a1cuRKSJGH+/PlyR3E4S5cuhSRJJrfQ0FBZsrA0dXBffPEF5s+fj5deegmHDh3CsGHDMG7cOBQUFMgdzaHU1NSgf//+WLt2rdxRHFZaWhpmz56NPXv2YOvWrWhqasLo0aNRU1MjdzSH0rlzZ6xatQoHDhzAgQMHcO+99+KBBx7A0aNH5Y7msPbv348PP/wQ/fr1kzuKw+rbty+KioqMt8OHD8uSg0sOdHCDBw/GwIED8f777xu39e7dG5MnT8bKlStlTOa4JEnCxo0bMXnyZLmjOLTS0lIEBwcjLS0NiYmJcsdxaP7+/li9ejVmzpwpdxSHU11djYEDB+K9997D8uXLMWDAALz99ttyx3IoS5cuxaZNm5CVlSV3FB5p6sgaGhqQmZmJ0aNHm2wfPXo0du3aJVMqIuvQarUALn2hk23o9XqkpKSgpqYG8fHxcsdxSLNnz8b48eMxcuRIuaM4tNzcXISHh6Nbt2545JFHcOrUKVly8IK9HVhZWRn0ej1CQkJMtoeEhKC4uFimVES3TgiBBQsWYOjQoYiJiZE7jsM5fPgw4uPjUVdXB29vb2zcuBF9+vSRO5bDSUlJwcGDB7F//365ozi0wYMH45///Ceio6Nx4cIFLF++HAkJCTh69CgCAgLaNQtLkx2QJMnkvhCixTYiezJnzhz89ttvyMjIkDuKQ+rVqxeysrJQWVmJDRs2YPr06UhLS2NxsqLCwkI888wz2LJlC9zd3eWO49DGjRtn/HNsbCzi4+PRo0cPfPLJJ1iwYEG7ZmFp6sACAwPh4uLS4qhSSUlJi6NPRPZi7ty5+Pbbb5Geno7OnTvLHcchubm5oWfPngCAuLg47N+/H3/5y1/wwQcfyJzMcWRmZqKkpASDBg0ybtPr9UhPT8fatWtRX18PFxcXGRM6Li8vL8TGxiI3N7fdX5tzmjowNzc3DBo0CFu3bjXZvnXrViQkJMiUiqhthBCYM2cOvv76a2zbtg3dunWTO5LTEEKgvr5e7hgO5b777sPhw4eRlZVlvMXFxWHatGnIyspiYbKh+vp6ZGdnIywsrN1fm0eaOrgFCxYgKSkJcXFxiI+Px4cffoiCggI8/fTTckdzKNXV1cjLyzPez8/PR1ZWFvz9/dGlSxcZkzmO2bNn47PPPsM333wDHx8f4xFUtVoNDw8PmdM5jsWLF2PcuHGIiIhAVVUVUlJSsGPHDqSmpsodzaH4+Pi0mI/n5eWFgIAAztOzsoULF2LixIno0qULSkpKsHz5cuh0OkyfPr3ds7A0dXBTpkxBeXk5Xn31VRQVFSEmJgbff/89IiMj5Y7mUA4cOIB77rnHeL/5PPn06dOxfv16mVI5luZlM0aMGGGyfd26dZgxY0b7B3JQFy5cQFJSEoqKiqBWq9GvXz+kpqZi1KhRckcjapOzZ8/i0UcfRVlZGYKCgjBkyBDs2bNHlu9BrtNEREREZAbOaSIiIiIyA0sTERERkRlYmoiIiIjMwNJEREREZAaWJiIiIiIzsDQRERERmYGliYiIiMgMLE1EREREZmBpIiKLnT59GpIkISsrS+4oRsePH8eQIUPg7u6OAQMGyB3HaVz738KOHTsgSRIqKytlzUVkCyxNRHZoxowZkCQJq1atMtm+adMmSJIkUyp5LVmyBF5eXjhx4gR+/vnn645p/twkSYJSqUT37t2xcOFC1NTUtHNax5WQkGC8hAuRo2FpIrJT7u7ueP3116HRaOSOYjUNDQ1tfuzJkycxdOhQREZGIiAgoNVxY8eORVFREU6dOoXly5fjvffew8KFC9v8utfS6/UwGAwttrf1vd3KZ2JNQgg0NTXddJybmxtCQ0OdtryTY2NpIrJTI0eORGhoKFauXNnqmKVLl7Y4VfX222+ja9euxvszZszA5MmTsWLFCoSEhMDPzw+vvPIKmpqa8Nxzz8Hf3x+dO3fGP/7xjxbPf/z4cSQkJMDd3R19+/bFjh07TPYfO3YM999/P7y9vRESEoKkpCSUlZUZ948YMQJz5szBggULEBgY2OpFZQ0GA1599VV07twZKpUKAwYMQGpqqnG/JEnIzMzEq6++CkmSsHTp0lY/E5VKhdDQUERERGDq1KmYNm0aNm3a1Or4hoYGPP/887jtttvg5eWFwYMHm7zP9evXw8/PD5s3b0afPn2gUqlw5swZdO3aFcuXL8eMGTOgVqsxa9YsAMCGDRvQt29fqFQqdO3aFW+++abJ613vcQ0NDZgzZw7CwsLg7u6Orl273vDvvfnv9JVXXkFwcDB8fX3x1FNPmRQwIQTeeOMNdO/eHR4eHujfvz+++uor4/7m02w//vgj4uLioFKpsHPnzlZf89rHNZ+ea/58fvzxR/Tu3Rve3t7G4nq1devWoXfv3nB3d8ftt9+O995776avRdTuBBHZnenTp4sHHnhAfP3118Ld3V0UFhYKIYTYuHGjuPp/1kuWLBH9+/c3eexbb70lIiMjTZ7Lx8dHzJ49Wxw/flx8/PHHAoAYM2aMeO2110ROTo5YtmyZUCqVoqCgQAghRH5+vgAgOnfuLL766itx7Ngx8cQTTwgfHx9RVlYmhBDi/PnzIjAwUCxatEhkZ2eLgwcPilGjRol77rnH+NrDhw8X3t7e4rnnnhPHjx8X2dnZ132/a9asEb6+vuLzzz8Xx48fF88//7xQKpUiJydHCCFEUVGR6Nu3r0hOThZFRUWiqqrqhp/b1ebOnSsCAgJa/aynTp0qEhISRHp6usjLyxOrV68WKpXK+Nrr1q0TSqVSJCQkiF9++UUcP35cVFdXi8jISOHr6ytWr14tcnNzRW5urjhw4IBQKBTi1VdfFSdOnBDr1q0THh4eYt26dcbXu97jVq9eLSIiIkR6ero4ffq02Llzp/jss89azTx9+nTh7e0tpkyZIo4cOSI2b94sgoKCxOLFi41jFi9eLG6//XaRmpoqTp48KdatWydUKpXYsWOHEEKI7du3CwCiX79+YsuWLSIvL8/4d3u15v8WDh06ZPI4jUZj8vmMHDlS7N+/X2RmZorevXuLqVOnGp/jww8/FGFhYWLDhg3i1KlTYsOGDcLf31+sX7++1fdIJAeWJiI7dPWX/5AhQ8Tjjz8uhGh7aYqMjBR6vd64rVevXmLYsGHG+01NTcLLy0t8/vnnQogrX5SrVq0yjmlsbBSdO3cWr7/+uhBCiD/96U9i9OjRJq9dWFgoAIgTJ04IIS6VpgEDBtz0/YaHh4vXXnvNZNudd94p/vjHPxrv9+/fXyxZsuSGz3Ntadq7d68ICAgQDz/88HXH5+XlCUmSxLlz50y233fffWLRokVCiEulAIDIysoyGRMZGSkmT55ssm3q1Kli1KhRJtuee+450adPnxs+bu7cueLee+8VBoPhhu/v6vfp7+8vampqjNvef/994e3tLfR6vaiurhbu7u5i165dJo+bOXOmePTRR4UQV8rPpk2bbvha5pQmACIvL8/4mHfffVeEhIQY70dERLQogcuWLRPx8fFmvV+i9uIqx9EtIrKe119/Hffeey+Sk5Pb/Bx9+/aFQnHlbH1ISAhiYmKM911cXBAQEICSkhKTx8XHxxv/7Orqiri4OGRnZwMAMjMzsX37dnh7e7d4vZMnTyI6OhoAEBcXd8NsOp0O58+fx913322y/e6778avv/5q5ju8YvPmzfD29kZTUxMaGxvxwAMP4J133rnu2IMHD0IIYczarL6+3mTelJubG/r169fi8de+t+zsbDzwwAMt3sfbb78NvV4PFxeX6z5uxowZGDVqFHr16oWxY8diwoQJGD169A3fZ//+/eHp6Wm8Hx8fj+rqahQWFqKkpAR1dXUtToc2NDTgjjvuuOF7aAtPT0/06NHDeD8sLMz431JpaSkKCwsxc+ZM4ylMAGhqauJkcupwWJqI7FxiYiLGjBmDxYsXY8aMGSb7FAoFhBAm2xobG1s8h1KpNLnf/Ouya7ddb4LztZonABsMBkycOBGvv/56izFhYWHGP3t5ed30Oa9+3mZCiDZNNr7nnnvw/vvvQ6lUIjw8vMX7vJrBYICLiwsyMzONhabZ1WXQw8PjulmufW/Xy3zt38/1Hjdw4EDk5+fjhx9+wE8//YSHH34YI0eONJmDZK6r/x6/++473HbbbSb7VSrVDbO0xfX+W2p+381ZPvroIwwePNhk3LWfOZHcWJqIHMCqVaswYMCAFkdEgoKCUFxcbPJlbc21lfbs2YPExEQAl44MZGZmYs6cOQAufdFv2LABXbt2hatr2/+p8fX1RXh4ODIyMoyvBQC7du3CXXfdZfHzeXl5oWfPnmaNveOOO6DX61FSUoJhw4ZZ/FrX6tOnDzIyMky27dq1C9HR0TctCL6+vpgyZQqmTJmC3/3udxg7diwqKirg7+9/3fG//voramtr4eHhAeDS35W3tzc6d+6MTp06QaVSoaCgAMOHD7/l93UrQkJCcNttt+HUqVOYNm2arFmIboalicgBxMbGYtq0aS1OM40YMQKlpaV444038Lvf/Q6pqan44Ycf4Ovra5XXfffddxEVFYXevXvjrbfegkajweOPPw4AmD17Nj766CM8+uijeO655xAYGIi8vDykpKTgo48+sugownPPPYclS5agR48eGDBgANatW4esrCx8+umnVnkfrYmOjsa0adPw2GOP4c0338Qdd9yBsrIybNu2DbGxsbj//vster7k5GTceeedWLZsGaZMmYLdu3dj7dq1N/2l2FtvvYWwsDAMGDAACoUCX375JUJDQ+Hn59fqYxoaGjBz5kz8v//3/3DmzBksWbIEc+bMgUKhgI+PDxYuXIhnn30WBoMBQ4cOhU6nw65du+Dt7Y3p06db9L5u1dKlSzFv3jz4+vpi3LhxqK+vx4EDB6DRaLBgwYJ2zUJ0IyxNRA5i2bJl+M9//mOyrXfv3njvvfewYsUKLFu2DA899BAWLlyIDz/80CqvuWrVKrz++us4dOgQevTogW+++QaBgYEAgPDwcPzyyy944YUXMGbMGNTX1yMyMhJjx441mT9ljnnz5kGn0yE5ORklJSXo06cPvv32W0RFRVnlfdzIunXrsHz5ciQnJ+PcuXMICAhAfHy8xYUJuHT07T//+Q9efvllLFu2DGFhYXj11VdbnFa9lre3N15//XXk5ubCxcUFd955J77//vsbfo733XcfoqKikJiYiPr6ejzyyCMmSzEsW7YMwcHBWLlyJU6dOgU/Pz8MHDgQixcvtvh93aonnngCnp6eWL16NZ5//nl4eXkhNjYW8+fPb/csRDciieudUCciIrs1Y8YMVFZW3nD9KSKyHBe3JCIiIjIDSxMRERGRGXh6joiIiMgMPNJEREREZAaWJiIiIiIzsDQRERERmYGliYiIiMgMLE1EREREZmBpIiIiIjIDSxMRERGRGViaiIiIiMzw/wHCQ12ls3sW9gAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "counts = df_final['p_anno_error_cnt'].value_counts()\n",
    "plt.xlabel(\"Number of P errors per line\")\n",
    "plt.ylabel(\"Number of Lines with Errors\")\n",
    "counts.plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NsMxzLUo-r97"
   },
   "source": [
    "extracting all the tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Yu5WXgR4ssjS"
   },
   "outputs": [],
   "source": [
    "df_final['p_annotator_error_tag'] = df_final['p_annotator_list'].str.findall('<(.*?)>')\n",
    "df_final['g_annotator_error_tag'] = df_final['g_annotator_list'].str.findall('<(.*?)>')\n",
    "# df_final_v2['g_annotator_error_tag'] = df_final_v2['g_annotator_list'].str.extractall('<(.*?)>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "A1cQ3k4HvkKz",
    "outputId": "a2c53699-2a32-40dd-f2f6-89b5dc73aabf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>is_duplicated</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>p_annotator_list</th>\n",
       "      <th>g_annotator_list</th>\n",
       "      <th>p_anno_error_cnt</th>\n",
       "      <th>g_anno_error_cnt</th>\n",
       "      <th>p_annotator_error_tag</th>\n",
       "      <th>g_annotator_error_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Una persona famosa que admiro es Lauren Jauregui</td>\n",
       "      <td>Una persona famosa que admiro es Lauren Jauregui</td>\n",
       "      <td>Una persona famosa que admiro es Lauren Jauregui</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Ella es una cantante en el grupo Fifth Harmony</td>\n",
       "      <td>Ella es una cantante en el grupo Fifth Harmony</td>\n",
       "      <td>Ella es una cantante en el grupo Fifth Harmony</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Ella ha asumido su papel de cantante que much...</td>\n",
       "      <td>Ella ha asumido su papel de cantante que much...</td>\n",
       "      <td>Ella ha asumido su papel de cantante que much...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Ella habla en contra del fanatismo y el odio ...</td>\n",
       "      <td>Ella habla en contra del fanatismo y el odio ...</td>\n",
       "      <td>Ella habla en contra del fanatismo y el odio ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>146362</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>Muchas niñas, especialmente las niñas que son...</td>\n",
       "      <td>Muchas niñas, especialmente las niñas que son...</td>\n",
       "      <td>Muchas niñas, especialmente las niñas que son...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>[]</td>\n",
       "      <td>[ga:fm:det:an, ga:fm:gat:noun:an, ga:fm:adj:an]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>Entonces, Hawaii es la casa a muchos frutas t...</td>\n",
       "      <td>Entonces, Hawaii es la casa a muchos frutas t...</td>\n",
       "      <td>Entonces, Hawaii es la casa a [muchos]{muchas...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>[]</td>\n",
       "      <td>[ga:fm:det:inan, ga:fm:adj:an]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>Hawaii es un especial isla y tiene algunos vo...</td>\n",
       "      <td>Hawaii es un especial isla y tiene algunos vo...</td>\n",
       "      <td>Hawaii es [un]{una}&lt;ga:fm:det:inan&gt; especial ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "      <td>[ga:fm:det:inan]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>EN la isla largo, tu veas (lava) ir en la oce...</td>\n",
       "      <td>EN la isla largo, tu veas (lava) ir en la oce...</td>\n",
       "      <td>EN la isla [largo]{larga}&lt;ga:fm:adj:inan&gt;, tu...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "      <td>[ga:fm:adj:inan]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>\\n\\nYo quiero ir a Hwaii con mi novio prque e...</td>\n",
       "      <td>\\n\\nYo quiero ir a Hwaii con mi novio prque e...</td>\n",
       "      <td>\\n\\nYo quiero ir a Hwaii con mi novio prque e...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>172450</td>\n",
       "      <td>1049</td>\n",
       "      <td>False</td>\n",
       "      <td>Yo soy muy emocionada que yo voy a Hawaii est...</td>\n",
       "      <td>Yo soy muy emocionada que yo voy a Hawaii est...</td>\n",
       "      <td>Yo soy muy emocionada que yo voy a Hawaii est...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "      <td>[na:ps:det:an]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22941 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  paragraph_id  is_duplicated  \\\n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "0    146362             1          False   \n",
       "..      ...           ...            ...   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "167  172450          1049          False   \n",
       "\n",
       "                                             essay_new  \\\n",
       "0     Una persona famosa que admiro es Lauren Jauregui   \n",
       "0       Ella es una cantante en el grupo Fifth Harmony   \n",
       "0     Ella ha asumido su papel de cantante que much...   \n",
       "0     Ella habla en contra del fanatismo y el odio ...   \n",
       "0     Muchas niñas, especialmente las niñas que son...   \n",
       "..                                                 ...   \n",
       "167   Entonces, Hawaii es la casa a muchos frutas t...   \n",
       "167   Hawaii es un especial isla y tiene algunos vo...   \n",
       "167   EN la isla largo, tu veas (lava) ir en la oce...   \n",
       "167   \\n\\nYo quiero ir a Hwaii con mi novio prque e...   \n",
       "167   Yo soy muy emocionada que yo voy a Hawaii est...   \n",
       "\n",
       "                                      p_annotator_list  \\\n",
       "0     Una persona famosa que admiro es Lauren Jauregui   \n",
       "0       Ella es una cantante en el grupo Fifth Harmony   \n",
       "0     Ella ha asumido su papel de cantante que much...   \n",
       "0     Ella habla en contra del fanatismo y el odio ...   \n",
       "0     Muchas niñas, especialmente las niñas que son...   \n",
       "..                                                 ...   \n",
       "167   Entonces, Hawaii es la casa a muchos frutas t...   \n",
       "167   Hawaii es un especial isla y tiene algunos vo...   \n",
       "167   EN la isla largo, tu veas (lava) ir en la oce...   \n",
       "167   \\n\\nYo quiero ir a Hwaii con mi novio prque e...   \n",
       "167   Yo soy muy emocionada que yo voy a Hawaii est...   \n",
       "\n",
       "                                      g_annotator_list  p_anno_error_cnt  \\\n",
       "0     Una persona famosa que admiro es Lauren Jauregui                 0   \n",
       "0       Ella es una cantante en el grupo Fifth Harmony                 0   \n",
       "0     Ella ha asumido su papel de cantante que much...                 0   \n",
       "0     Ella habla en contra del fanatismo y el odio ...                 0   \n",
       "0     Muchas niñas, especialmente las niñas que son...                 0   \n",
       "..                                                 ...               ...   \n",
       "167   Entonces, Hawaii es la casa a [muchos]{muchas...                 0   \n",
       "167   Hawaii es [un]{una}<ga:fm:det:inan> especial ...                 0   \n",
       "167   EN la isla [largo]{larga}<ga:fm:adj:inan>, tu...                 0   \n",
       "167   \\n\\nYo quiero ir a Hwaii con mi novio prque e...                 0   \n",
       "167   Yo soy muy emocionada que yo voy a Hawaii est...                 0   \n",
       "\n",
       "     g_anno_error_cnt p_annotator_error_tag  \\\n",
       "0                   0                    []   \n",
       "0                   0                    []   \n",
       "0                   0                    []   \n",
       "0                   0                    []   \n",
       "0                   3                    []   \n",
       "..                ...                   ...   \n",
       "167                 2                    []   \n",
       "167                 1                    []   \n",
       "167                 1                    []   \n",
       "167                 0                    []   \n",
       "167                 1                    []   \n",
       "\n",
       "                               g_annotator_error_tag  \n",
       "0                                                 []  \n",
       "0                                                 []  \n",
       "0                                                 []  \n",
       "0                                                 []  \n",
       "0    [ga:fm:det:an, ga:fm:gat:noun:an, ga:fm:adj:an]  \n",
       "..                                               ...  \n",
       "167                   [ga:fm:det:inan, ga:fm:adj:an]  \n",
       "167                                 [ga:fm:det:inan]  \n",
       "167                                 [ga:fm:adj:inan]  \n",
       "167                                               []  \n",
       "167                                   [na:ps:det:an]  \n",
       "\n",
       "[22941 rows x 10 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ERdIy-NEtv1_"
   },
   "outputs": [],
   "source": [
    "df_p_tags_exploded = df_final.explode('p_annotator_error_tag')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "id": "aQl5avHTu3VY",
    "outputId": "f16c301f-90a7-4c74-b642-b6d8bd148ffd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: xlabel='P error annotation', ylabel='Occurences'>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAIHCAYAAAB5bp2NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSbUlEQVR4nO3deXxOZ/7/8fedRIJIYglZKoJaikRrq23se5WiQ1tatGo6Qy21lapWW5WOqaI11WmLqKXRzlQ3aieqQUnFTomdRFoiITSJ5Pz+8HN/3U1skTsnzv16Ph7nMb2vc93n/py5b8fbda5zjs0wDEMAAAAW5WZ2AQAAAM5E2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJbmYXYBhUF2drZOnz4tHx8f2Ww2s8sBAAC3wTAMXbhwQcHBwXJzu/H4DWFH0unTpxUSEmJ2GQAAIA9OnDih8uXL33A9YUeSj4+PpKv/Z/n6+ppcDQAAuB2pqakKCQmx/z1+I4QdyX7qytfXl7ADAMA95lZTUJigDAAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALI2wAwAALM3D7AKsoOLYpaZ99tF3Opv22QAA3AsY2QEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZWaMJORESEbDabhg8fbm8zDEMTJ05UcHCwihUrppYtW2rPnj0O70tPT9eQIUPk7+8vb29vde3aVSdPnizg6gEAQGFVKMLO1q1b9fHHH6t27doO7VOmTNF7772nmTNnauvWrQoMDFS7du104cIFe5/hw4dryZIlioqK0saNG3Xx4kU9+uijysrKKujdAAAAhZDpYefixYvq06ePPvnkE5UqVcrebhiGpk+frvHjx6tHjx4KCwvTvHnzdOnSJS1atEiSlJKSotmzZ2vq1Klq27at6tSpowULFmjXrl1avXq1WbsEAAAKEdPDzuDBg9W5c2e1bdvWof3IkSNKTExU+/bt7W1eXl5q0aKFYmJiJEmxsbHKzMx06BMcHKywsDB7n9ykp6crNTXVYQEAANbkYeaHR0VF6ZdfftHWrVtzrEtMTJQkBQQEOLQHBATo2LFj9j6enp4OI0LX+lx7f24iIiL0xhtv3G35Lq/i2KWmffbRdzqb9tkAgHuLaSM7J06c0LBhw7RgwQIVLVr0hv1sNpvDa8MwcrT92a36jBs3TikpKfblxIkTd1Y8AAC4Z5gWdmJjY5WUlKR69erJw8NDHh4eio6O1vvvvy8PDw/7iM6fR2iSkpLs6wIDA5WRkaHk5OQb9smNl5eXfH19HRYAAGBNpoWdNm3aaNeuXYqLi7Mv9evXV58+fRQXF6fKlSsrMDBQq1atsr8nIyND0dHRatKkiSSpXr16KlKkiEOfhIQE7d69294HAAC4NtPm7Pj4+CgsLMyhzdvbW2XKlLG3Dx8+XJMnT1bVqlVVtWpVTZ48WcWLF1fv3r0lSX5+fhowYIBGjhypMmXKqHTp0ho1apTCw8NzTHgGAACuydQJyrcyZswYXb58WYMGDVJycrIaNmyolStXysfHx95n2rRp8vDwUK9evXT58mW1adNGkZGRcnd3N7FyAABQWNgMwzDMLsJsqamp8vPzU0pKSp7m77jqVUmuut8AgMLhdv/+Nv0+OwAAAM5E2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZG2AEAAJZmatiZNWuWateuLV9fX/n6+qpx48b64Ycf7Ov79+8vm83msDRq1MhhG+np6RoyZIj8/f3l7e2trl276uTJkwW9KwAAoJAyNeyUL19e77zzjrZt26Zt27apdevWeuyxx7Rnzx57n44dOyohIcG+LFu2zGEbw4cP15IlSxQVFaWNGzfq4sWLevTRR5WVlVXQuwMAAAohDzM/vEuXLg6v3377bc2aNUubN29WrVq1JEleXl4KDAzM9f0pKSmaPXu25s+fr7Zt20qSFixYoJCQEK1evVodOnRw7g4AAIBCr9DM2cnKylJUVJTS0tLUuHFje/v69etVrlw5VatWTQMHDlRSUpJ9XWxsrDIzM9W+fXt7W3BwsMLCwhQTE3PDz0pPT1dqaqrDAgAArMn0sLNr1y6VKFFCXl5e+vvf/64lS5aoZs2akqROnTpp4cKFWrt2raZOnaqtW7eqdevWSk9PlyQlJibK09NTpUqVcthmQECAEhMTb/iZERER8vPzsy8hISHO20EAAGAqU09jSVL16tUVFxen8+fP63//+5/69eun6Oho1axZU0888YS9X1hYmOrXr6/Q0FAtXbpUPXr0uOE2DcOQzWa74fpx48ZpxIgR9tepqakEHgAALMr0sOPp6akqVapIkurXr6+tW7dqxowZ+s9//pOjb1BQkEJDQ3Xw4EFJUmBgoDIyMpScnOwwupOUlKQmTZrc8DO9vLzk5eWVz3sCAAAKI9NPY/2ZYRj201R/dvbsWZ04cUJBQUGSpHr16qlIkSJatWqVvU9CQoJ2795907ADAABch6kjO6+88oo6deqkkJAQXbhwQVFRUVq/fr2WL1+uixcvauLEiXr88ccVFBSko0eP6pVXXpG/v7+6d+8uSfLz89OAAQM0cuRIlSlTRqVLl9aoUaMUHh5uvzoLAAC4NlPDzpkzZ/TMM88oISFBfn5+ql27tpYvX6527drp8uXL2rVrlz777DOdP39eQUFBatWqlRYvXiwfHx/7NqZNmyYPDw/16tVLly9fVps2bRQZGSl3d3cT9wwAABQWpoad2bNn33BdsWLFtGLFiltuo2jRovrggw/0wQcf5GdpAADAIgrdnB0AAID8RNgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWZmrYmTVrlmrXri1fX1/5+vqqcePG+uGHH+zrDcPQxIkTFRwcrGLFiqlly5bas2ePwzbS09M1ZMgQ+fv7y9vbW127dtXJkycLelcAAEAhZWrYKV++vN555x1t27ZN27ZtU+vWrfXYY4/ZA82UKVP03nvvaebMmdq6dasCAwPVrl07Xbhwwb6N4cOHa8mSJYqKitLGjRt18eJFPfroo8rKyjJrtwAAQCFiatjp0qWLHnnkEVWrVk3VqlXT22+/rRIlSmjz5s0yDEPTp0/X+PHj1aNHD4WFhWnevHm6dOmSFi1aJElKSUnR7NmzNXXqVLVt21Z16tTRggULtGvXLq1evdrMXQMAAIVEoZmzk5WVpaioKKWlpalx48Y6cuSIEhMT1b59e3sfLy8vtWjRQjExMZKk2NhYZWZmOvQJDg5WWFiYvU9u0tPTlZqa6rAAAABrMj3s7Nq1SyVKlJCXl5f+/ve/a8mSJapZs6YSExMlSQEBAQ79AwIC7OsSExPl6empUqVK3bBPbiIiIuTn52dfQkJC8nmvAABAYWF62Klevbri4uK0efNm/eMf/1C/fv20d+9e+3qbzebQ3zCMHG1/dqs+48aNU0pKin05ceLE3e0EAAAotEwPO56enqpSpYrq16+viIgIPfjgg5oxY4YCAwMlKccITVJSkn20JzAwUBkZGUpOTr5hn9x4eXnZrwC7tgAAAGsyPez8mWEYSk9PV6VKlRQYGKhVq1bZ12VkZCg6OlpNmjSRJNWrV09FihRx6JOQkKDdu3fb+wAAANfmYeaHv/LKK+rUqZNCQkJ04cIFRUVFaf369Vq+fLlsNpuGDx+uyZMnq2rVqqpataomT56s4sWLq3fv3pIkPz8/DRgwQCNHjlSZMmVUunRpjRo1SuHh4Wrbtq2ZuwYAAAoJU8POmTNn9MwzzyghIUF+fn6qXbu2li9frnbt2kmSxowZo8uXL2vQoEFKTk5Ww4YNtXLlSvn4+Ni3MW3aNHl4eKhXr166fPmy2rRpo8jISLm7u5u1WwAAoBCxGYZhmF2E2VJTU+Xn56eUlJQ8zd+pOHapE6q6PUff6WzaZ7vqfgMACofb/fu70M3ZAQAAyE+EHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGmEHQAAYGn5EnaysrIUFxen5OTk/NgcAABAvslT2Bk+fLhmz54t6WrQadGiherWrauQkBCtX78+P+sDAAC4K3kKO//973/14IMPSpK+++47HTlyRPv379fw4cM1fvz4fC0QAADgbuQp7Pz+++8KDAyUJC1btkw9e/ZUtWrVNGDAAO3atStfCwQAALgbeQo7AQEB2rt3r7KysrR8+XK1bdtWknTp0iW5u7vna4EAAAB3wyMvb3r22WfVq1cvBQUFyWazqV27dpKkLVu26IEHHsjXAgEAAO5GnsLOxIkTFRYWphMnTqhnz57y8vKSJLm7u2vs2LH5WiAAAMDdyFPYkaS//vWvkqQ//vjD3tavX7+7rwgAACAf5WnOTlZWlt566y3dd999KlGihA4fPixJmjBhgv2SdAAAgMIgT2Hn7bffVmRkpKZMmSJPT097e3h4uD799NPb3k5ERIQaNGggHx8flStXTt26ddOBAwcc+vTv3182m81hadSokUOf9PR0DRkyRP7+/vL29lbXrl118uTJvOwaAACwmDyFnc8++0wff/yx+vTp43D1Ve3atbV///7b3k50dLQGDx6szZs3a9WqVbpy5Yrat2+vtLQ0h34dO3ZUQkKCfVm2bJnD+uHDh2vJkiWKiorSxo0bdfHiRT366KPKysrKy+4BAAALydOcnVOnTqlKlSo52rOzs5WZmXnb21m+fLnD67lz56pcuXKKjY1V8+bN7e1eXl72+/r8WUpKimbPnq358+fbL4FfsGCBQkJCtHr1anXo0OG26wEAANaTp5GdWrVq6ccff8zR/uWXX6pOnTp5LiYlJUWSVLp0aYf29evXq1y5cqpWrZoGDhyopKQk+7rY2FhlZmaqffv29rbg4GCFhYUpJiYm189JT09XamqqwwIAAKwpTyM7r7/+up555hmdOnVK2dnZ+uqrr3TgwAF99tln+v777/NUiGEYGjFihP7yl78oLCzM3t6pUyf17NlToaGhOnLkiCZMmKDWrVsrNjZWXl5eSkxMlKenp0qVKuWwvYCAACUmJub6WREREXrjjTfyVCcAALi35Glkp0uXLlq8eLGWLVsmm82m1157Tfv27dN3331nv8HgnXrxxRe1c+dOff755w7tTzzxhDp37qywsDB16dJFP/zwg3799VctXbr0ptszDEM2my3XdePGjVNKSop9OXHiRJ5qBgAAhV+e77PToUOHfJsPM2TIEH377bfasGGDypcvf9O+QUFBCg0N1cGDByVJgYGBysjIUHJyssPoTlJSkpo0aZLrNry8vOw3QgQAANaWp5GdrVu3asuWLTnat2zZom3btt32dgzD0IsvvqivvvpKa9euVaVKlW75nrNnz+rEiRMKCgqSJNWrV09FihTRqlWr7H0SEhK0e/fuG4YdAADgOvIUdgYPHpzrqZ9Tp05p8ODBd7SdBQsWaNGiRfLx8VFiYqISExN1+fJlSdLFixc1atQobdq0SUePHtX69evVpUsX+fv7q3v37pIkPz8/DRgwQCNHjtSaNWu0fft2Pf300woPD7dfnQUAAFxXnk5j7d27V3Xr1s3RXqdOHe3du/e2tzNr1ixJUsuWLR3a586dq/79+8vd3V27du3SZ599pvPnzysoKEitWrXS4sWL5ePjY+8/bdo0eXh4qFevXrp8+bLatGmjyMhInsAOAADyFna8vLx05swZVa5c2aE9ISFBHh63v0nDMG66vlixYlqxYsUtt1O0aFF98MEH+uCDD277swEAgGvI02msdu3a2a9ouub8+fN65ZVX8nw1FgAAgDPkaWRn6tSpat68uUJDQ+03EYyLi1NAQIDmz5+frwUCAADcjTyFnfvuu087d+7UwoULtWPHDhUrVkzPPvusnnrqKRUpUiS/awQAAMizPN9nx9vbW3/729/ysxYAAIB8l+ew8+uvv2r9+vVKSkpSdna2w7rXXnvtrgsDAADID3kKO5988on+8Y9/yN/fX4GBgQ6PZbj2+AgAAIDCIE9hZ9KkSXr77bf18ssv53c9AAAA+SpPl54nJyerZ8+e+V0LAABAvstT2OnZs6dWrlyZ37UAAADkuzydxqpSpYomTJigzZs3Kzw8PMfl5kOHDs2X4gAAAO5WnsLOxx9/rBIlSig6OlrR0dEO62w2G2EHAAAUGnkKO0eOHMnvOgAAAJwiT3N2rsnIyNCBAwd05cqV/KoHAAAgX+Up7Fy6dEkDBgxQ8eLFVatWLR0/flzS1bk677zzTr4WCAAAcDfyFHbGjRunHTt2aP369SpatKi9vW3btlq8eHG+FQcAAHC38jRn5+uvv9bixYvVqFEjh7sn16xZU/Hx8flWHAAAwN3K08jOb7/9pnLlyuVoT0tLcwg/AAAAZstT2GnQoIGWLl1qf30t4HzyySdq3Lhx/lQGAACQD/J0GisiIkIdO3bU3r17deXKFc2YMUN79uzRpk2bctx3BwAAwEx5Gtlp0qSJYmJidOnSJd1///1auXKlAgICtGnTJtWrVy+/awQAAMizOx7ZyczM1N/+9jdNmDBB8+bNc0ZNAAAA+eaOR3aKFCmiJUuWOKMWAACAfJen01jdu3fX119/nc+lAAAA5L88P/X8rbfeUkxMjOrVqydvb2+H9TwIFAAAFBZ5CjuffvqpSpYsqdjYWMXGxjqs46nnAACgMOGp5wAAwNLu6qnnAAAAhV2eRnaee+65m66fM2dOnooBAADIb3kKO8nJyQ6vMzMztXv3bp0/f16tW7fOl8IAAADyQ57CTm732cnOztagQYNUuXLluy4KAAAgv+TbnB03Nze99NJLmjZtWn5tEgAA4K7l6wTl+Ph4Xbly5bb7R0REqEGDBvLx8VG5cuXUrVs3HThwwKGPYRiaOHGigoODVaxYMbVs2VJ79uxx6JOenq4hQ4bI399f3t7e6tq1q06ePJkv+wQAAO5teTqNNWLECIfXhmEoISFBS5cuVb9+/W57O9HR0Ro8eLAaNGigK1euaPz48Wrfvr327t1rv1HhlClT9N577ykyMlLVqlXTpEmT1K5dOx04cEA+Pj6SpOHDh+u7775TVFSUypQpo5EjR+rRRx9VbGys3N3d87KLAADAIvIUdrZv3+7w2s3NTWXLltXUqVNveaXW9ZYvX+7weu7cuSpXrpxiY2PVvHlzGYah6dOna/z48erRo4ckad68eQoICNCiRYv0wgsvKCUlRbNnz9b8+fPVtm1bSdKCBQsUEhKi1atXq0OHDnnZRQAAYBF5Cjvr1q3L7zokSSkpKZKk0qVLS7p688LExES1b9/e3sfLy0stWrRQTEyMXnjhBcXGxiozM9OhT3BwsMLCwhQTE5Nr2ElPT1d6err9dWpqqlP2BwAAmC9Pc3aOHDmigwcP5mg/ePCgjh49mqdCDMPQiBEj9Je//EVhYWGSpMTERElSQECAQ9+AgAD7usTERHl6eqpUqVI37PNnERER8vPzsy8hISF5qhkAABR+eQo7/fv3V0xMTI72LVu2qH///nkq5MUXX9TOnTv1+eef51hns9kcXhuGkaPtz27WZ9y4cUpJSbEvJ06cyFPNAACg8MtT2Nm+fbuaNm2ao71Ro0aKi4u74+0NGTJE3377rdatW6fy5cvb2wMDAyUpxwhNUlKSfbQnMDBQGRkZOW50eH2fP/Py8pKvr6/DAgAArClPYcdms+nChQs52lNSUpSVlXXb2zEMQy+++KK++uorrV27VpUqVXJYX6lSJQUGBmrVqlX2toyMDEVHR6tJkyaSpHr16qlIkSIOfRISErR79257HwAA4LryNEG5WbNmioiI0Oeff26/tDsrK0sRERH6y1/+ctvbGTx4sBYtWqRvvvlGPj4+9hEcPz8/FStWTDabTcOHD9fkyZNVtWpVVa1aVZMnT1bx4sXVu3dve98BAwZo5MiRKlOmjEqXLq1Ro0YpPDzcfnUWAABwXXkKO1OmTFHz5s1VvXp1NWvWTJL0448/KjU1VWvXrr3t7cyaNUuS1LJlS4f2uXPn2uf+jBkzRpcvX9agQYOUnJyshg0bauXKlfZ77EjStGnT5OHhoV69euny5ctq06aNIiMjuccOAACQzTAMIy9vPH36tGbOnKkdO3aoWLFiql27tl588UX7ZeP3ktTUVPn5+SklJSVP83cqjl3qhKpuz9F3Opv22a663wCAwuF2//7O08iOdPVeNpMnT87r2wEAAApEniYoz507V19++WWO9i+//FLz5s2766IAAADyS57CzjvvvCN/f/8c7eXKlWO0BwAAFCp5CjvHjh3LcZm4JIWGhur48eN3XRQAAEB+yVPYKVeunHbu3JmjfceOHSpTpsxdFwUAAJBf8hR2nnzySQ0dOlTr1q1TVlaWsrKytHbtWg0bNkxPPvlkftcIAACQZ3m6GmvSpEk6duyY2rRpIw+Pq5vIyspSv379mLMDAAAKlTyFHU9PTy1evFijRo3SkSNHVLx4cYWHhys0NDS/6wMAALgrdxx2zp8/r/Hjx2vx4sX2h2+WKlVKTz75pCZNmqSSJUvmd40AAAB5dkdh59y5c2rcuLFOnTqlPn36qEaNGjIMQ/v27VNkZKTWrFmjmJgYlSpVyln1AgAA3JE7CjtvvvmmPD09FR8fr4CAgBzr2rdvrzfffFPTpk3L1yIBAADy6o6uxvr666/17rvv5gg6khQYGKgpU6ZoyZIl+VYcAADA3bqjsJOQkKBatWrdcH1YWJgSExPvuigAAID8ckdhx9/fX0ePHr3h+iNHjnBTQQAAUKjcUdjp2LGjxo8fr4yMjBzr0tPTNWHCBHXs2DHfigMAALhbdzRB+Y033lD9+vVVtWpVDR48WA888IAkae/evfrwww+Vnp6u+fPnO6VQAACAvLijsFO+fHlt2rRJgwYN0rhx42QYhiTJZrOpXbt2mjlzpkJCQpxSKAAAQF7c8U0FK1WqpB9++EHJyck6ePCgJKlKlSoqXbp0vhcHAABwt/L0uAjp6l2TH3744fysBQAAIN/l6annAAAA9wrCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDTCDgAAsDRTw86GDRvUpUsXBQcHy2az6euvv3ZY379/f9lsNoelUaNGDn3S09M1ZMgQ+fv7y9vbW127dtXJkycLcC8AAEBhZmrYSUtL04MPPqiZM2fesE/Hjh2VkJBgX5YtW+awfvjw4VqyZImioqK0ceNGXbx4UY8++qiysrKcXT4AALgHeJj54Z06dVKnTp1u2sfLy0uBgYG5rktJSdHs2bM1f/58tW3bVpK0YMEChYSEaPXq1erQoUO+1wwAAO4thX7Ozvr161WuXDlVq1ZNAwcOVFJSkn1dbGysMjMz1b59e3tbcHCwwsLCFBMTc8NtpqenKzU11WEBAADWVKjDTqdOnbRw4UKtXbtWU6dO1datW9W6dWulp6dLkhITE+Xp6alSpUo5vC8gIECJiYk33G5ERIT8/PzsS0hIiFP3AwAAmMfU01i38sQTT9j/OywsTPXr11doaKiWLl2qHj163PB9hmHIZrPdcP24ceM0YsQI++vU1FQCDwAAFlWoR3b+LCgoSKGhoTp48KAkKTAwUBkZGUpOTnbol5SUpICAgBtux8vLS76+vg4LAACwpnsq7Jw9e1YnTpxQUFCQJKlevXoqUqSIVq1aZe+TkJCg3bt3q0mTJmaVCQAAChFTT2NdvHhRhw4dsr8+cuSI4uLiVLp0aZUuXVoTJ07U448/rqCgIB09elSvvPKK/P391b17d0mSn5+fBgwYoJEjR6pMmTIqXbq0Ro0apfDwcPvVWQAAwLWZGna2bdumVq1a2V9fm0fTr18/zZo1S7t27dJnn32m8+fPKygoSK1atdLixYvl4+Njf8+0adPk4eGhXr166fLly2rTpo0iIyPl7u5e4PsDAAAKH1PDTsuWLWUYxg3Xr1ix4pbbKFq0qD744AN98MEH+VkaAACwiHtqzg4AAMCdIuwAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLMzXsbNiwQV26dFFwcLBsNpu+/vprh/WGYWjixIkKDg5WsWLF1LJlS+3Zs8ehT3p6uoYMGSJ/f395e3ura9euOnnyZAHuBQAAKMxMDTtpaWl68MEHNXPmzFzXT5kyRe+9955mzpyprVu3KjAwUO3atdOFCxfsfYYPH64lS5YoKipKGzdu1MWLF/Xoo48qKyuroHYDAAAUYh5mfninTp3UqVOnXNcZhqHp06dr/Pjx6tGjhyRp3rx5CggI0KJFi/TCCy8oJSVFs2fP1vz589W2bVtJ0oIFCxQSEqLVq1erQ4cOBbYvAACgcCq0c3aOHDmixMREtW/f3t7m5eWlFi1aKCYmRpIUGxurzMxMhz7BwcEKCwuz98lNenq6UlNTHRYAAGBNhTbsJCYmSpICAgIc2gMCAuzrEhMT5enpqVKlSt2wT24iIiLk5+dnX0JCQvK5egAAUFgU2rBzjc1mc3htGEaOtj+7VZ9x48YpJSXFvpw4cSJfagUAAIVPoQ07gYGBkpRjhCYpKck+2hMYGKiMjAwlJyffsE9uvLy85Ovr67AAAABrKrRhp1KlSgoMDNSqVavsbRkZGYqOjlaTJk0kSfXq1VORIkUc+iQkJGj37t32PgAAwLWZejXWxYsXdejQIfvrI0eOKC4uTqVLl1aFChU0fPhwTZ48WVWrVlXVqlU1efJkFS9eXL1795Yk+fn5acCAARo5cqTKlCmj0qVLa9SoUQoPD7dfnQUAAFybqWFn27ZtatWqlf31iBEjJEn9+vVTZGSkxowZo8uXL2vQoEFKTk5Ww4YNtXLlSvn4+NjfM23aNHl4eKhXr166fPmy2rRpo8jISLm7uxf4/gAAgMLHZhiGYXYRZktNTZWfn59SUlLyNH+n4tilTqjq9hx9p7Npn+2q+w0AKBxu9+/vQjtnBwAAID8QdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKUV6rAzceJE2Ww2hyUwMNC+3jAMTZw4UcHBwSpWrJhatmypPXv2mFgxAAAobAp12JGkWrVqKSEhwb7s2rXLvm7KlCl67733NHPmTG3dulWBgYFq166dLly4YGLFAACgMCn0YcfDw0OBgYH2pWzZspKujupMnz5d48ePV48ePRQWFqZ58+bp0qVLWrRokclVAwCAwqLQh52DBw8qODhYlSpV0pNPPqnDhw9Lko4cOaLExES1b9/e3tfLy0stWrRQTEzMTbeZnp6u1NRUhwUAAFhToQ47DRs21GeffaYVK1bok08+UWJiopo0aaKzZ88qMTFRkhQQEODwnoCAAPu6G4mIiJCfn599CQkJcdo+AAAAcxXqsNOpUyc9/vjjCg8PV9u2bbV06VJJ0rx58+x9bDabw3sMw8jR9mfjxo1TSkqKfTlx4kT+Fw8AAAoFD7MLuBPe3t4KDw/XwYMH1a1bN0lSYmKigoKC7H2SkpJyjPb8mZeXl7y8vJxZKiys4tilpn320Xc6m/bZAHCvKtQjO3+Wnp6uffv2KSgoSJUqVVJgYKBWrVplX5+RkaHo6Gg1adLExCoBAEBhUqhHdkaNGqUuXbqoQoUKSkpK0qRJk5Samqp+/frJZrNp+PDhmjx5sqpWraqqVatq8uTJKl68uHr37m126QAAoJAo1GHn5MmTeuqpp/T777+rbNmyatSokTZv3qzQ0FBJ0pgxY3T58mUNGjRIycnJatiwoVauXCkfHx+TKwcAAIVFoQ47UVFRN11vs9k0ceJETZw4sWAKAgAA95x7as4OAADAnSLsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAAS/MwuwAA94aKY5ea9tlH3+ls2mcDuPcxsgMAACzNMmHnww8/VKVKlVS0aFHVq1dPP/74o9klAQCAQsASYWfx4sUaPny4xo8fr+3bt6tZs2bq1KmTjh8/bnZpAADAZJYIO++9954GDBig559/XjVq1ND06dMVEhKiWbNmmV0aAAAw2T0/QTkjI0OxsbEaO3asQ3v79u0VExOT63vS09OVnp5uf52SkiJJSk1NzVMN2emX8vS+/JDXmvMD+13w2O+CF/b6CtM+e/cbHUz7bPa74LHfd+7ascEwjJt3NO5xp06dMiQZP/30k0P722+/bVSrVi3X97z++uuGJBYWFhYWFhYLLCdOnLhpVrjnR3ausdlsDq8Nw8jRds24ceM0YsQI++vs7GydO3dOZcqUueF7nCU1NVUhISE6ceKEfH19C/SzzcR+s9+ugP1mv12BmfttGIYuXLig4ODgm/a758OOv7+/3N3dlZiY6NCelJSkgICAXN/j5eUlLy8vh7aSJUs6q8Tb4uvr61J/OK5hv10L++1a2G/XYtZ++/n53bLPPT9B2dPTU/Xq1dOqVasc2letWqUmTZqYVBUAACgs7vmRHUkaMWKEnnnmGdWvX1+NGzfWxx9/rOPHj+vvf/+72aUBAACTWSLsPPHEEzp79qzefPNNJSQkKCwsTMuWLVNoaKjZpd2Sl5eXXn/99Ryn1ayO/Wa/XQH7zX67gnthv22GcavrtQAAAO5d9/ycHQAAgJsh7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAgDvy448/6umnn1bjxo116tQpSdL8+fO1ceNGkyvLnSXus3OvysjIUFJSkrKzsx3aK1SoYFJFzpWWlqZ33nlHa9asyXW/Dx8+bFJlzvfrr79q/fr1ue73a6+9ZlJVzuXK37cr47hm/d/5//73Pz3zzDPq06ePtm/frvT0dEnShQsXNHnyZC1btszkCnMi7Jjg4MGDeu655xQTE+PQfu3hpVlZWSZV5lzPP/+8oqOj9cwzzygoKKjAH7pqlk8++UT/+Mc/5O/vr8DAQIf9ttlslg07rvp9S1J8fLzmzp2r+Ph4zZgxQ+XKldPy5csVEhKiWrVqmV2eU3Bcc53f+aRJk/TRRx+pb9++ioqKsrc3adJEb775pomV3Rg3FTRB06ZN5eHhobFjx+b6h+PBBx80qTLnKlmypJYuXaqmTZuaXUqBCg0N1aBBg/Tyyy+bXUqBctXvOzo6Wp06dVLTpk21YcMG7du3T5UrV9aUKVP0888/67///a/ZJToFxzXX+Z0XL15ce/fuVcWKFeXj46MdO3aocuXKOnz4sGrWrKk//vjD7BJzYGTHBHFxcYqNjdUDDzxgdikFqlSpUipdurTZZRS45ORk9ezZ0+wyCpyrft9jx47VpEmTNGLECPn4+NjbW7VqpRkzZphYmXNxXHMdQUFBOnTokCpWrOjQvnHjRlWuXNmcom6BCcomqFmzpn7//Xezyyhwb731ll577TVdunTJ7FIKVM+ePbVy5Uqzyyhwrvp979q1S927d8/RXrZsWZ09e9aEigoGxzXX+Z2/8MILGjZsmLZs2SKbzabTp09r4cKFGjVqlAYNGmR2ebliZMcE//znPzVmzBhNnjxZ4eHhKlKkiMN6X19fkypzrqlTpyo+Pl4BAQGqWLFijv3+5ZdfTKrMuapUqaIJEyZo8+bNuX7fQ4cONaky53LV77tkyZJKSEhQpUqVHNq3b9+u++67z6SqnI/jmuv8zseMGaOUlBS1atVKf/zxh5o3by4vLy+NGjVKL774otnl5Yo5OyZwc7s6oPbnc9pWn8j3xhtv3HT966+/XkCVFKw//6V3PZvNZsmrNSTX/b7HjBmjTZs26csvv1S1atX0yy+/6MyZM+rbt6/69u1r2f3muJY7q37fknTp0iXt3btX2dnZqlmzpkqUKGF2STdE2DFBdHT0Tde3aNGigCoBkN8yMzPVv39/RUVFyTAMeXh4KCsrS71791ZkZKTc3d3NLtEpOK65jueee04zZsxwmJMmXb0Mf8iQIZozZ45Jld0YYQcAnCA+Pl7bt29Xdna26tSpo6pVq5pdEpAv3N3dlZCQoHLlyjm0//777woMDNSVK1dMquzGmLNjokuXLun48ePKyMhwaK9du7ZJFTlXVlaWpk2bpi+++CLX/T537pxJlTnfyZMn9e233+a63++9955JVTmXK3/fknT//ffr/vvvN7uMAsdxzbq/89TUVBmGIcMwdOHCBRUtWtS+LisrS8uWLcsRgAoLwo4JfvvtNz377LP64Ycfcl1v5XPbn376qUaMGKEJEyZo/PjxOnr0qL7++mvL3lhPktasWaOuXbuqUqVKOnDggMLCwnT06FEZhqG6deuaXZ7TuNL3PWLEiNvua9Vwy3HN+r/zkiVLymazyWazqVq1ajnW22y2W85hMo2BAte7d2+jSZMmxs8//2x4e3sbK1euNObPn29Ur17d+P77780uz2kqV65s378SJUoYhw4dMgzDMGbMmGE89dRTZpbmVA0aNDAmTJhgGMbV/Y6PjzcuXLhgdO3a1fjwww9Nrs55XOn7btmypcPi4+NjFC9e3KhTp45Rp04dw9vb2/D19TVatWpldqlOw3HN+r/z9evXG+vWrTNsNpvx1VdfGevXr7cvMTExxqlTp8wu8YYIOyYIDAw0tmzZYhiGYfj4+BgHDhwwDMMwvvnmG6Np06ZmluZUxYsXN44dO2YYxtX/D2JjYw3DMIz4+HjD19fXzNKc6voDYMmSJY3du3cbhmEYcXFxRmhoqImVOZerft9Tp041unTpYpw7d87edu7cOeOxxx4z3n33XRMrcy6Oa67zOz969KiRnZ2d67pr/18UNtxU0ARpaWn285qlS5fWb7/9JkkKDw+35D0ZrilfvrwSEhIkXb33zLUb7W3dulVeXl5mluZU3t7e9gflBQcHKz4+3r7Oyjdhc9Xve+rUqYqIiFCpUqXsbaVKldKkSZM0depUEytzLo5rrvM7r1y5sv37vd7Zs2dveqsNMxF2TFC9enUdOHBAkvTQQw/pP//5j06dOqWPPvpIQUFBJlfnPN27d9eaNWskScOGDdOECRNUtWpV9e3bV88995zJ1TlPo0aN9NNPP0mSOnfurJEjR+rtt9/Wc889p0aNGplcnfO46vedmpqqM2fO5GhPSkrShQsXTKioYHBcc53fuXGDi7gvXrzoMGm5MOHScxMsXLjQfi+O7du3q0OHDjp79qw8PT0VGRmpJ554wuwSC8SWLVv0008/qUqVKuratavZ5TjN4cOHdfHiRdWuXVuXLl3SqFGjtHHjRlWpUkXTpk1TaGio2SUWCFf5vvv27avo6GhNnTrVHmY3b96s0aNHq3nz5po3b57JFToHx7WrrPw7vzYRf8aMGRo4cKCKFy9uX5eVlaUtW7bI3d3d/o+7woSwUwhcunRJ+/fvV4UKFeTv7292OQDuwrVAO2fOHGVmZkqSPDw8NGDAAP3rX/+St7e3yRUWDI5r1tOqVStJV28g2bhxY3l6etrXeXp6qmLFiho1alShvKcUYQcoIBkZGUpKSlJ2drZDe4UKFUyqCM6Ulpam+Ph4GYahKlWquEzIgfU9++yzmjFjxj31vDPm7Jhkw4YN2rZtm0Pbtm3btGHDBpMqKhhubm6qVauWQ1uNGjUsewt9Sfr111/VrFkzFStWTKGhoapUqZIqVaqkihUrFtrJfPnFFb/va7y9vVW7dm09+OCDLhN0OK79Hyv/zufOnStfX18dOnRIK1as0OXLlyXdeC5PYcBNBU3SsmVLPfDAA9q7d6+97ZlnntGvv/5q2ZtvSdKcOXNUsmRJh7aIiAilpKSYU1ABePbZZ+Xh4aHvv/9eQUFBOR6UaGWu+H1LV6/E+fLLL3O9o+5XX31lUlXOx3Ht/1j5d37u3Dn17NlT69atk81m08GDB1W5cmU9//zzKlmyZKG86pDTWCY5duyYihQpouDgYHvb6dOnlZmZ6TITVl2Ft7e3YmNj9cADD5hdCgpAVFSU+vbtq/bt22vVqlVq3769Dh48qMTERHXv3l1z5841u0Sn4bjmGvr27aukpCR9+umnqlGjhnbs2KHKlStr5cqVeumll7Rnzx6zS8yBkR2T5PYH//oDhNWdPHlSNptN9913n9mlOF3NmjUtfT+d2+FK3/fkyZM1bdo0DR48WD4+PpoxY4YqVaqkF154wdKXYEsc11zld75y5UqtWLFC5cuXd2ivWrWqjh07ZlJVN8ecHRPFxsZqwYIFWrhwoaVvunVNdna23nzzTfn5+Sk0NFQVKlRQyZIl9dZbb+WYtGsl//znPzVmzBitX79eZ8+eVWpqqsNiVa76fcfHx6tz586SJC8vL6Wlpclms+mll17Sxx9/bHJ1zsdxzfq/87S0NIfLzq/5/fffC++NFE26c7NLO3PmjNGqVSvDZrMZpUqVMkqWLGnYbDajdevWRlJSktnlOc3YsWONsmXLGh9++KGxY8cOIy4uzvj3v/9tlC1b1njllVfMLs9pbDabYbPZDDc3N4flWptVuer3Xb58eWPnzp2GYRhG7dq1jUWLFhmGYRgxMTGWfXyAYXBcc6Xf+SOPPGK8+uqrhmFcfRzO4cOHjaysLKNnz57G448/bnJ1uSPsmKBXr15GvXr1jL1799rb9uzZY9SvX9948sknTazMuYKCgoxvvvkmR/vXX39tBAcHm1BRwbj+YXm5LVblqt/3U089ZUydOtUwDMOYNGmSUbZsWeP55583QkNDje7du5tcnfNwXHNk5d/5nj17jLJlyxodO3Y0PD09jb/+9a9GjRo1jICAAPtzAAsbJiibwM/PT6tXr1aDBg0c2n/++We1b99e58+fN6cwJytatKh27typatWqObQfOHBADz30kP3yRViDq37f586d0x9//KHg4GBlZ2fr3Xfftd8xe8KECQ7PzLISjmuu9TtPTEzUrFmzFBsbq+zsbNWtW1eDBw8utPPSmKBsguzsbBUpUiRHe5EiRSx7jleSHnzwQc2cOVPvv/++Q/vMmTP14IMPmlSVc+zcuVNhYWFyc3PTzp07b9q3du3aBVRVwXKl7/t6pUuXtv+3m5ubxowZozFjxphYUcHguOZav/PAwEC98cYbZpdx2xjZMcFjjz2m8+fP6/PPP7dfqXDq1Cn16dNHpUqV0pIlS0yu0Dmio6PVuXNnVahQQY0bN5bNZlNMTIxOnDihZcuWqVmzZmaXmG/c3NyUmJiocuXKyc3NTTabLdcbbtlsNsvef8SVvu8/y87O1qFDh3K9Y3bz5s1Nqsq5OK5Z+3d+q3+0Xa8w/gOOsGOCEydO6LHHHtPu3bsVEhIim82m48ePKzw8XN98802Oy/ms5PTp0/r3v/+t/fv3yzAM1axZU4MGDbLc5anHjh1ThQoVZLPZbnkpppXvP+Iq3/f1Nm/erN69e+vYsWM5Aq6Vwy3HNWv/zm/2j7brFdbfOGHHRKtWrXL4w9G2bVuzSwJwlx566CFVq1ZNb7zxRq53zPbz8zOpsoLBcc2a7uT+OYXxH3CEHTjVvT70iTvDXKWrd8zesWOHqlSpYnYpcBKOa/cewk4B+fPktZsZOnSoEyspWNcPfV7/L9xrP7vr2wrj0Gd+cXNzU40aNRxuo16jRg3LPTOIuUpS69atNWbMGHXs2NHsUpzu/fff19/+9jcVLVr0lsc4jmvWERERoYCAAD333HMO7XPmzNFvv/2ml19+2aTKboywU0D+/HTr3377TZcuXbI/PO78+fMqXry4ypUrp8OHD5tQoXNcP/S5fft2jRo1SqNHj1bjxo0lSZs2bdLUqVM1ZcoUdevWzaQqnS8yMlIlS5Z02Mevv/5aKSkp6tevn3mF5TPmKklLlizRq6++qtGjRys8PDzHFUpW+pd+pUqVtG3bNpUpUybHMe56NpuN45qFVKxYUYsWLVKTJk0c2rds2aInn3xSR44cMamymyig+/ngOgsXLjSaNm1q7N+/3962f/9+o1mzZsaCBQtMrMy5GjRoYCxdujRH+9KlS426deuaUBGQ/67dMfv6xRXumO2qXPG45uXlZRw+fDhHe3x8vOHl5WVCRbfGfXZMMGHCBP33v/9V9erV7W3Vq1fXtGnT9Ne//lV9+vQxsTrn2bVrV67/+qtUqZL27t1rQkUFz1UeFOjKCuW/auE0rnhcCwkJ0U8//ZRjv3/66adCewUaDwI1QUJCgjIzM3O0Z2Vl6cyZMyZUVDBq1KihSZMm6Y8//rC3paena9KkSapRo4aJlTmXKz4o8Bo3NzfVqlXLoa1GjRpyd3c3qSLnCw0NveliZRs2bNC2bdsc2rZt26YNGzaYVJHzueJx7fnnn9fw4cM1d+5cHTt2TMeOHdOcOXP00ksvaeDAgWaXlytGdkzQpk0bDRw4ULNnz1a9evVks9m0bds2vfDCC5a+TPOjjz5Sly5dFBISYr+z6I4dO2Sz2fT999+bXJ3zjB8/XrNnz9Y777yjpk2byjAM/fTTT5o4caL++OMPvf3222aX6DRz5syxz0u7JiIiQikpKeYUVEDi4+M1ffp07du3TzabTTVq1NCwYcN0//33m12aU7Vs2VIPPPCAw4jGM888Y7mJ+NdzxePamDFjdO7cOQ0aNEgZGRmSrj424+WXX9a4ceNMri53TFA2wW+//aZ+/fpp+fLl9smLV65cUYcOHRQZGaly5cqZXKHzXLp0SQsWLHC4D0fv3r3l7e1tdmlOExwcrI8++khdu3Z1aP/mm280aNAgnTp1yqTK4AwrVqxQ165d9dBDD9nDbUxMjHbs2KHvvvtO7dq1M7tEpzl27JiKFCnicCrj9OnTyszMtPSolise1yTp4sWL2rdvn4oVK6aqVavKy8vL7JJuiLBjooMHD2rfvn0yDEM1atTI8SA5WIOrPijws88+U/369VWzZk2H9j/++ENffPGF+vbta1JlzlWnTh116NBB77zzjkP72LFjtXLlSv3yyy8mVQbkj8jISD3xxBMqVqyY2aXcNsIO4GQNGzZUw4YNc9yHZMiQIdq6das2b95sUmXO5ebmJm9vb0VGRurxxx+3t585c0bBwcGWPa1RtGhR7dq1S1WrVnVo//XXX1W7dm2HuR3AvSgoKEhpaWnq2bOnBgwYkOMS9MKICcomccWJfJJrTlidMmWK5syZo5o1a2rAgAF6/vnnVbNmTUVGRupf//qX2eU51RtvvKFnnnlGEydONLuUAlO2bFnFxcXlaI+Li7P0KeqsrCy9++67evjhhxUYGKjSpUs7LFbmase1kydPasGCBUpOTlarVq30wAMP6J///KcSExPNLu2GmKBsElecyCe55oTVFi1a6Ndff3V4UGCPHj0s96DA3Dz99NNq0qSJunfvrt27d2v+/Plml+R0AwcO1N/+9jcdPnxYTZo0kc1m08aNG/XPf/5TI0eONLs8p3njjTf06aefasSIEZowYYLGjx+vo0eP6uuvv9Zrr71mdnlO5WrHNXd3d3Xt2lVdu3ZVUlKSFixYoMjISE2YMEEdO3bUgAED1KVLF7m5FZ7xFE5jmcRVJ/LBdbi7uyshIUHlypXT8ePH1bVrV9lsNn300Udq0qSJZUO9YRiaPn26pk6dqtOnT0u6Okl99OjRGjp0aI4Hg1rF/fffr/fff1+dO3eWj4+P4uLi7G2bN2/WokWLzC4RTrJlyxbNmTNH8+bNU1BQkM6fP6+SJUtq7ty5atmypdnlSSLsoAC50oRVHhTo+Jws6eoVK3369NGaNWuUlpZm2bBzvQsXLkiSfHx8TK7E+by9vbVv3z5VqFBBQUFBWrp0qerWravDhw+rTp06lh3lcFVnzpzR/PnzNXfuXB0+fFjdunXTgAED1LZtW12+fFmvvvqq/vvf/97R09KdibCDAuNKE1Zd/UGB0tXTGqNHj1bx4sUd2l9//XVt2LBB69atM6kyOEP16tX12WefqWHDhmrWrJk6d+6ssWPHavHixRoyZIiSkpLMLtFptm7dqi+//FLHjx+333fmmq+++sqkqpynS5cuWrFihapVq6bnn39effv2zTEv6/Tp0ypfvnyhuXEqc3ZMkJWVpWnTpumLL77I9Q/HuXPnTKrM+a5NWN21a5elJ61e/8iAWz0o0Kpef/11SdLevXsdfuf16tVTvXr1zCzNqc6ePavXXntN69atU1JSUo6DvVX/fHfv3l1r1qxRw4YNNWzYMD311FOaPXu2jh8/rpdeesns8pwmKipKffv2Vfv27bVq1Sq1b99eBw8eVGJiorp37252eU5Rrlw5RUdH249nuQkKCipUj05hZMcEr7322k0n8g0dOtTsEp3i2mmNw4cPq3v37mratKnmz5+v1NRUy43sXO/hhx/WxIkT9cgjjzi0L1u2TBMmTFBsbKxJlTnX4cOH1aNHD+3cudM+yiX936iWVb/vTp06KT4+XgMGDFBAQECOOTpWesr9zWzZskU//fSTqlSpkuOGmlZSu3ZtvfDCCxo8eLB8fHy0Y8cOVapUSS+88IKCgoL0xhtvmF0iRNgxhatO5HPVCavFihXTL7/8kuM5Ofv27VPdunUte1PBLl26yN3dXZ988okqV66sn3/+WWfPntXIkSP17rvvqlmzZmaX6BQ+Pj7auHGj/dEBrubPI3nXWDXweHt7a8+ePapYsaL8/f21bt06hYeHa9++fWrdurUSEhLMLjHfDR06VFWqVMnxD/OZM2fq0KFDmj59ujmF3UxBPV4d/6d48eLGsWPHDMMwjMDAQCM2NtYwDMOIj483fH19zSzNqWw2m3HmzBn767S0NKNbt26Gj4+P4ebmZmJlzlWnTh2jd+/exuXLl+1tf/zxh9G7d2+jTp06JlbmXGXKlDF27NhhGIZh+Pr6Gvv37zcMwzDWrFljPPTQQ2aW5lT169c3Nm3aZHYZBS4+Pt6oXbu2YbPZDDc3N/v/Xlusqnz58sbOnTsNwzCM2rVrG4sWLTIMwzBiYmIsezwPDg42tm3blqM9NjbWuO+++0yo6NYKz0XwLqR8+fL2tF+lShWtXLlS0tVJboX52SJ36/XXX1eJEiXsr4sXL64lS5bopZdeUvPmzU2szLk++ugjrV69WiEhIWrbtq3atm2r8uXLa9WqVfroo4/MLs9psrKy7N+3v7+//TLs0NBQHThwwMzSnOrDDz/U+PHjFR0drbNnzyo1NdVhsaphw4apUqVKOnPmjIoXL67du3crOjpa9evX1/r1680uz2maNWumVatWSZJ69eqlYcOGaeDAgXrqqafUunVrk6tzjrNnz8rPzy9Hu6+vr37//XcTKro1JiibwFUn8rnqhNWHH35YR44ccXhQ4BNPPGH5BwWGhYVp586dqly5sho2bKgpU6bI09NTH3/8sSpXrmx2eU5TsmRJpaSk5PiLzvj/V+ZZ9XTtpk2btHbtWpUtW1Zubm5yd3fXX/7yF0VERGjo0KHavn272SU6xcyZM+2PABk3bpyKFCmijRs3qkePHpa9mWKVKlW0fPlyvfjiiw7tP/zwQ6H9s82cnULAVSbyueqEVVe1YsUKpaWlqUePHjp8+LAeffRR7d+/X2XKlNHixYst+6/ehx9+WB4eHho2bFiuE5RbtGhhUmXOVapUKcXGxqpy5cq6//779emnn6pVq1aKj49XeHi4Ll26ZHaJTtG6dWu1aNHC/o+5a5KTk/X4449r7dq1JlXmPHPmzNGLL76o0aNH2/8cr1mzRlOnTtX06dM1cOBAkyvMibBjIlebyOeqE1bxf86dO6dSpUpZ9i7C0tXTs9u3b1f16tXNLqVANWvWTCNHjlS3bt3Uu3dvJScn69VXX9XHH3+s2NhY7d692+wSncLNzU1lypRR06ZNtXDhQvtorRXvH3a9WbNm6e2337afnq5YsaImTpxYeG8Oa+J8IZflqhP5XHXCqmFcnZxds2ZNh7YHHnjA0t+3q2rWrJmxatUqs8socMuXLzf+97//GYZx9RhXo0YNw2azGf7+/saaNWtMrs55bDabERcXZzRs2NAICwszjhw5YhiGYSQmJrrEn++kpCTjwoULZpdxS8zZMcG1iXyrV69W5cqVtWXLFp07d84+wmFVuU1YrV69uuUnrEqu96BAVzZkyBANGzZMo0ePVnh4uIoUKeKw3qqPB+nQoYP9vytXrqy9e/e6xEiedPUGetHR0XruuefUoEEDffnllzluNWFVZcuWNbuE28JpLBP4+/tr7dq1ql27tvz8/PTzzz+revXqWrt2rUaOHGnZiXyuOswN15Lbk56vf3SIVU9ruKrr7x8mSZMmTdKkSZP08ssva9KkSZb5vuvWras1a9aoVKlSqlOnzk0D7C+//FKAld0eRnZM4KojHK+++qrS0tIkXT0gPProo2rWrJl9wipgBYXpFvlwvj+PF7z66quqUaOG5e6U/dhjj9lvjdKtWzdzi8kDRnZMwAjH/3GVYW5Xe1Ag4CqOHTumChUq5DiG7dmzR9u2bbNc6LlXcVNBE7z66qv2hwNOmjRJx44dU7NmzbRs2TK9//77JldXsEqXLm35oBMVFaWmTZtq7969WrJkiTIzM7V3716tXbs21xtz4d43f/58NW3aVMHBwTp27Jgkafr06frmm29Mrgz5LTQ0NNdjWK1atSwfdLZt26b58+drwYIFhf4Zf4zsFBKuMsLhinhQoGuZNWuWXnvtNQ0fPlxvv/22du/ercqVKysyMlLz5s3TunXrzC4RuCsnT57UU089pZ9++sl+4cX58+fVpEkTff755woJCTG3wFwwslNIuMIIh6uKj49X586dJUleXl5KS0uTzWbTSy+9pI8//tjk6pDfPvjgA33yyScaP3683N3d7e3169fXrl27TKwMyB/PPfecMjMztW/fPp07d07nzp3Tvn37ZBiGBgwYYHZ5uWKCMuBkpUuX1oULFyRJ9913n3bv3q3w8HCdP3/esneVdWVHjhxRnTp1crRfC7rAve7HH39UTEyMw40zq1evrg8++EBNmzY1sbIbY2QHcDJXfFCgK6tUqZLi4uJytP/www+qWbNmwRcE5LMKFSooMzMzR/uVK1d03333mVDRrTGyAziZKz4o0JWNHj1agwcP1h9//CHDMPTzzz/r888/V0REhD799FOzywPu2pQpUzRkyBD9+9//Vr169WSz2bRt2zYNGzas0N4YlwnKgJO54oMCXd0nn3yiSZMm6cSJE5Kunr6cOHFioZ3PANzKny+gSUtL05UrV+ThcXXM5Np/e3t769y5c2aVeUOEHcDJXPVBgZB+//13ZWdnq1y5ckpLS1NsbKyaN29udlnAHZs3b95t9y2Ml9wTdgAnc3Nz0/bt2/XCCy8oLS1N3333nSpWrEjYcTE7duxQ3bp1+b4BEzBBGSgA1x4UWLt2bTVo0EDr1683uyQAcBmEHcDJrp3n9vLy0sKFCzVs2DB17NhRH374ocmVAYBr4GoswMlc5UGBAFBYEXYAJzty5IjKli3r0Pb444/rgQce0LZt20yqCvnt22+/vel6noYOmIcJygCQD9zcbj0rwGazMUEZMAFhBwAA3JGtW7fqyy+/1PHjx5WRkeGw7quvvjKpqhtjgjIA5JNLly7d8GGfe/bs0cWLFwu4IiD/RUVFqWnTptq7d6+WLFmizMxM7d27V2vXrpWfn5/Z5eWKsAMA+SQjI0MNGzbUzz//7NC+d+9e1alTh7ADS5g8ebKmTZum77//Xp6enpoxY4b27dunXr16qUKFCmaXlyvCDgDkk5IlS6pLly457jY7f/58tW3bVoGBgSZVBuSf+Ph4de7cWdLVW2qkpaXJZrPppZde0scff2xydbkj7ABAPurbt6+++OILXblyRdLVWw8sXLhQ/fv3N7cwIJ+ULl1aFy5ckHT1uW+7d++WJJ0/f16XLl0ys7QbIuwAQD7q2LGjPDw8tHTpUknSunXrdPHiRXXr1s3cwoB80qxZM61atUqS1KtXLw0bNkwDBw7UU089pTZt2phcXe64GgsA8tno0aMVHx+vr776Ss8++6yKFi2qWbNmmV0WkC/OnTunP/74Q8HBwcrOzta7776rjRs3qkqVKpowYYJKlSpldok5EHYAIJ/t2rVLDz/8sA4dOqQaNWpo5cqVatSokdllAU6Vlpam2NhYNW/e3OxSciDsAIAT1KtXTz4+PkpMTNT+/fvNLgdwuh07dqhu3bqF8saZzNkBACfo16+fNmzYwDPQgEKAZ2MBgBM8/fTTSk5O1rPPPmt2KYDL4zQWAAC4a4X5NBYjOwAA4Ja+/fbbm64/cuRIAVVy5xjZAQAAt+TmdutpvjabrVCO7BB2AACApXE1FgAAsDTCDgAAuC1paWnatWtXruv27NmjixcvFnBFt4ewAwAAbktmZqYaNmyon3/+2aF97969qlOnDmEHAADc20qWLKkuXbpo3rx5Du3z589X27ZtFRgYaFJlN0fYAQAAt61v37764osvdOXKFUmSYRhauHCh+vfvb25hN0HYAQAAt61jx47y8PDQ0qVLJUnr1q3TxYsX1a1bN3MLuwnCDgAAuG3u7u56+umn7aey5s+fryeeeEKenp4mV3Zj3GcHAADckV27dunhhx/WoUOHVKNGDa1cuVKNGjUyu6wbIuwAAIA7Vq9ePfn4+CgxMVH79+83u5yb4jQWAAC4Y/369dOGDRvUr18/s0u5JR4ECgAA7tjTTz+t5ORkPfvss2aXckucxgIAAJbGaSwAAGBphB0AAGBphB0AAGBphB0AAGBphB0AKEQmTpyohx56yOwyAEsh7AAupH///rLZbLLZbCpSpIgqV66sUaNGKS0tzezS7llHjx6VzWZTXFzcHb/XZrPp66+/dmgbNWqU1qxZkz/FAZDEfXYAl9OxY0fNnTtXmZmZ+vHHH/X8888rLS1Ns2bNypftZ2VlyWazyc3N8d9SGRkZeXp2Tl7fd68qUaKESpQoYXYZgKUwsgO4GC8vLwUGBiokJES9e/dWnz59cowuXC8jI0NjxozRfffdJ29vbzVs2FDr16+3r4+MjFTJkiX1/fffq2bNmvLy8tKxY8dUsWJFTZo0Sf3795efn58GDhwoSfrf//6nWrVqycvLSxUrVtTUqVMdPu9G7/uz5cuX6y9/+YtKliypMmXK6NFHH1V8fLx9/bURl6+++kqtWrVS8eLF9eCDD2rTpk05al+xYoVq1KihEiVKqGPHjkpISLD3yc7O1ptvvqny5cvLy8tLDz30kJYvX25fX6lSJUlSnTp1ZLPZ1LJlS0nS1q1b1a5dO/n7+8vPz08tWrTQL7/84rCfktS9e3fZbDb76z+fxrrV59/OfgKujrADuLhixYopMzPzhuufffZZ/fTTT4qKitLOnTvVs2dPdezYUQcPHrT3uXTpkiIiIvTpp59qz549KleunCTpX//6l8LCwhQbG6sJEyYoNjZWvXr10pNPPqldu3Zp4sSJmjBhgiIjIx0+88/vy01aWppGjBihrVu3as2aNXJzc1P37t2VnZ3t0G/8+PEaNWqU4uLiVK1aNT311FO6cuWKQ+3vvvuu5s+frw0bNuj48eMaNWqUff2MGTM0depUvfvuu9q5c6c6dOigrl272vf/559/liStXr1aCQkJ+uqrryRJFy5cUL9+/fTjjz9q8+bNqlq1qh555BFduHBB0tUwJElz585VQkKC/fWf3erzb3c/AZdmAHAZ/fr1Mx577DH76y1bthhlypQxevXqlWv/Q4cOGTabzTh16pRDe5s2bYxx48YZhmEYc+fONSQZcXFxDn1CQ0ONbt26ObT17t3baNeunUPb6NGjjZo1a970fbcjKSnJkGTs2rXLMAzDOHLkiCHJ+PTTT+199uzZY0gy9u3b51D7oUOH7H3+/e9/GwEBAfbXwcHBxttvv+3wWQ0aNDAGDRrk8Dnbt2+/aX1XrlwxfHx8jO+++87eJslYsmSJQ7/XX3/dePDBB+/482+2n4CrY2QHcDHff/+9SpQooaJFi6px48Zq3ry5Pvjgg1z7/vLLLzIMQ9WqVbPPJSlRooSio6MdThl5enqqdu3aOd5fv359h9f79u1T06ZNHdqaNm2qgwcPKisr64bvy018fLx69+6typUry9fX13466fjx4w79rq8rKChIkpSUlGRvK168uO6//36HPtfWp6am6vTp07nWvG/fvpvWl5SUpL///e+qVq2a/Pz85Ofnp4sXL+ao72bu5PNvtZ+AK2OCMuBiWrVqpVmzZqlIkSIKDg5WkSJFbtg3Oztb7u7uio2Nlbu7u8O66yfRFitWTDabLcf7vb29HV4bhpGjn5HL4/n+/L7cdOnSRSEhIfrkk08UHBys7OxshYWFKSMjw6Hf9ft37bOvP9X15/232Ww5asqt5tz293r9+/fXb7/9punTpys0NFReXl5q3Lhxjvpux+18/q32E3BlhB3AxXh7e6tKlSq31bdOnTrKyspSUlKSmjVrdtefXbNmTW3cuNGhLSYmRtWqVcsRpm7m7Nmz2rdvn/7zn//Y6/rzdvODr6+vgoODtXHjRjVv3tyh5ocffliS7FeKXT8yJUk//vijPvzwQz3yyCOSpBMnTuj333936FOkSJEc77vTzwdwa4QdADdUrVo19enTR3379tXUqVNVp04d/f7771q7dq3Cw8Ptf5HfrpEjR6pBgwZ666239MQTT2jTpk2aOXOmPvzwwzvaTqlSpVSmTBl9/PHHCgoK0vHjxzV27Ng72sbtGj16tF5//XXdf//9euihhzR37lzFxcVp4cKFkqRy5cqpWLFiWr58ucqXL6+iRYvKz89PVapU0fz581W/fn2lpqZq9OjRKlasmMO2K1asqDVr1qhp06by8vJSqVKl7vjzAdwac3YA3NTcuXPVt29fjRw5UtWrV1fXrl21ZcsWhYSE3PG26tatqy+++EJRUVEKCwvTa6+9pjfffFP9+/e/o+24ubkpKipKsbGxCgsL00svvaR//etfd1zP7Rg6dKhGjhypkSNHKjw8XMuXL9e3336rqlWrSpI8PDz0/vvv6z//+Y+Cg4P12GOPSZLmzJmj5ORk1alTR88884yGDh1qv0rtmqlTp2rVqlUKCQlRnTp18vT5AG7NZuR2whwAAMAiGNkBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACW9v8ALbXMp5fVMrAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "counts = df_p_tags_exploded.p_annotator_error_tag.value_counts()\n",
    "plt.ylabel(\"Occurences\")\n",
    "plt.xlabel(\"P error annotation\")\n",
    "counts.plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aK9xItJMO3MO",
    "outputId": "1ee20bf2-2c78-45a9-c36a-9e091d0acc32"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>is_duplicated</th>\n",
       "      <th>essay_new</th>\n",
       "      <th>p_annotator_list</th>\n",
       "      <th>g_annotator_list</th>\n",
       "      <th>p_anno_error_cnt</th>\n",
       "      <th>g_anno_error_cnt</th>\n",
       "      <th>p_annotator_error_tag</th>\n",
       "      <th>g_annotator_error_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>127886</td>\n",
       "      <td>44</td>\n",
       "      <td>False</td>\n",
       "      <td>\\n\\nEl álbum más reciente &lt;&lt;Lemonade&gt;&gt; es sob...</td>\n",
       "      <td>\\n\\nEl álbum más reciente &lt;&lt;Lemonade&gt;&gt; es sob...</td>\n",
       "      <td>\\n\\nEl álbum más reciente &lt;&lt;Lemonade&gt;&gt; es sob...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;Lemonade</td>\n",
       "      <td>[&lt;Lemonade]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>127886</td>\n",
       "      <td>589</td>\n",
       "      <td>False</td>\n",
       "      <td>Por las mañanas iríamos al cafe se llama &lt;&lt;La...</td>\n",
       "      <td>Por las mañanas iríamos al cafe se llama &lt;&lt;La...</td>\n",
       "      <td>Por las mañanas iríamos al cafe se llama &lt;&lt;La...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;La bicyclette</td>\n",
       "      <td>[&lt;La bicyclette]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  paragraph_id  is_duplicated  \\\n",
       "64  127886            44          False   \n",
       "57  127886           589          False   \n",
       "\n",
       "                                            essay_new  \\\n",
       "64   \\n\\nEl álbum más reciente <<Lemonade>> es sob...   \n",
       "57   Por las mañanas iríamos al cafe se llama <<La...   \n",
       "\n",
       "                                     p_annotator_list  \\\n",
       "64   \\n\\nEl álbum más reciente <<Lemonade>> es sob...   \n",
       "57   Por las mañanas iríamos al cafe se llama <<La...   \n",
       "\n",
       "                                     g_annotator_list  p_anno_error_cnt  \\\n",
       "64   \\n\\nEl álbum más reciente <<Lemonade>> es sob...                 0   \n",
       "57   Por las mañanas iríamos al cafe se llama <<La...                 0   \n",
       "\n",
       "    g_anno_error_cnt p_annotator_error_tag g_annotator_error_tag  \n",
       "64                 0             <Lemonade           [<Lemonade]  \n",
       "57                 0        <La bicyclette      [<La bicyclette]  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p_tags_exploded[(df_p_tags_exploded.p_annotator_error_tag == '<Lemonade') | (df_p_tags_exploded.p_annotator_error_tag == '<La bicyclette')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Q9al4iTUwxlf"
   },
   "outputs": [],
   "source": [
    "df_g_tags_exploded = df_final.explode('g_annotator_error_tag')\n",
    "# counts = df_g_tags_exploded.g_annotator_error_tag.value_counts()\n",
    "# counts.plot.bar()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eHjMoCwqO3MP",
    "outputId": "d60cde4b-3778-4267-addf-2396da947b5f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: xlabel='G error (first tag)', ylabel='Occurences'>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAIHCAYAAAB+LEOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIBUlEQVR4nO3deViU9eL//9eIgoo4isqmiJiGmlimpWi5r7/UOp1P2rGIzGwzjdQWTx/LytQst5OfrCw3tDRP2qaRlku54JbkboqYSyAuMAgqINy/P7qcbyNgMCI3eD8f1zXXxbzv9wyve04nXr3vZWyGYRgCAACwsApmBwAAADAbhQgAAFgehQgAAFgehQgAAFgehQgAAFgehQgAAFgehQgAAFgehQgAAFheRbMDlBd5eXn6448/5OPjI5vNZnYcAABQBIZh6Ny5cwoKClKFCoWvA1GIiuiPP/5QcHCw2TEAAIAbjh07pnr16hW6nUJURD4+PpL+/ECrV69uchoAAFAU6enpCg4Odv4dLwyFqIguHyarXr06hQgAgHLm70534aRqAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRQiAABgeRXNDmA1DV5ebnYEF0cm3mN2BAAATMcKEQAAsDwKEQAAsDwKEQAAsDwKEQAAsDwKEQAAsDwKEQAAsDwKEQAAsDxTC9HMmTPVokULVa9eXdWrV1dERIS+++4753bDMDR27FgFBQWpSpUq6tSpk/bs2ePyHllZWRo2bJhq164tb29v9evXT8ePH3eZk5qaqsjISNntdtntdkVGRiotLa00dhEAAJQDphaievXqaeLEidq2bZu2bdumLl266N5773WWnkmTJmnKlCmaMWOGtm7dqoCAAHXv3l3nzp1zvkd0dLSWLVumRYsWaf369crIyFCfPn2Um5vrnDNw4EDFx8crNjZWsbGxio+PV2RkZKnvLwAAKJtshmEYZof4K19fX73zzjt67LHHFBQUpOjoaL300kuS/lwN8vf319tvv60nn3xSDodDderUUUxMjAYMGCBJ+uOPPxQcHKwVK1aoZ8+e2rdvn5o1a6a4uDi1adNGkhQXF6eIiAjt379fYWFhRcqVnp4uu90uh8Oh6tWru71/3KkaAIDSU9S/32XmHKLc3FwtWrRImZmZioiIUGJiopKTk9WjRw/nHC8vL3Xs2FEbN26UJG3fvl05OTkuc4KCgtS8eXPnnE2bNslutzvLkCS1bdtWdrvdOacgWVlZSk9Pd3kAAIAbk+mFaNeuXapWrZq8vLz01FNPadmyZWrWrJmSk5MlSf7+/i7z/f39nduSk5Pl6empmjVrXnWOn59fvt/r5+fnnFOQCRMmOM85stvtCg4Ovqb9BAAAZZfphSgsLEzx8fGKi4vT008/raioKO3du9e53Wazucw3DCPf2JWunFPQ/L97n9GjR8vhcDgfx44dK+ouAQCAcsb0QuTp6alGjRqpdevWmjBhgm699VZNnz5dAQEBkpRvFSclJcW5ahQQEKDs7GylpqZedc7Jkyfz/d5Tp07lW336Ky8vL+fVb5cfAADgxmR6IbqSYRjKyspSaGioAgICtGrVKue27OxsrVu3Tu3atZMktWrVSpUqVXKZk5SUpN27dzvnREREyOFwaMuWLc45mzdvlsPhcM4BAADWVtHMX/7vf/9bvXv3VnBwsM6dO6dFixZp7dq1io2Nlc1mU3R0tMaPH6/GjRurcePGGj9+vKpWraqBAwdKkux2uwYPHqyRI0eqVq1a8vX11ahRoxQeHq5u3bpJkpo2bapevXppyJAh+vDDDyVJTzzxhPr06VPkK8wAAMCNzdRCdPLkSUVGRiopKUl2u10tWrRQbGysunfvLkl68cUXdeHCBT3zzDNKTU1VmzZttHLlSvn4+DjfY+rUqapYsaL69++vCxcuqGvXrpo7d648PDyccxYuXKjhw4c7r0br16+fZsyYUbo7CwAAyqwydx+isor7EAEAUP6Uu/sQAQAAmIVCBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALI9CBAAALM/UQjRhwgTdcccd8vHxkZ+fn+677z4dOHDAZc6jjz4qm83m8mjbtq3LnKysLA0bNky1a9eWt7e3+vXrp+PHj7vMSU1NVWRkpOx2u+x2uyIjI5WWlna9dxEAAJQDphaidevWaejQoYqLi9OqVat06dIl9ejRQ5mZmS7zevXqpaSkJOdjxYoVLtujo6O1bNkyLVq0SOvXr1dGRob69Omj3Nxc55yBAwcqPj5esbGxio2NVXx8vCIjI0tlPwEAQNlW0cxfHhsb6/J8zpw58vPz0/bt29WhQwfnuJeXlwICAgp8D4fDoU8++UQxMTHq1q2bJGnBggUKDg7WDz/8oJ49e2rfvn2KjY1VXFyc2rRpI0maNWuWIiIidODAAYWFhV2nPQQAAOVBmTqHyOFwSJJ8fX1dxteuXSs/Pz/dfPPNGjJkiFJSUpzbtm/frpycHPXo0cM5FhQUpObNm2vjxo2SpE2bNslutzvLkCS1bdtWdrvdOedKWVlZSk9Pd3kAAIAbU5kpRIZhaMSIEbrrrrvUvHlz53jv3r21cOFCrV69WpMnT9bWrVvVpUsXZWVlSZKSk5Pl6empmjVruryfv7+/kpOTnXP8/Pzy/U4/Pz/nnCtNmDDBeb6R3W5XcHBwSe0qAAAoY0w9ZPZXzz77rHbu3Kn169e7jA8YMMD5c/PmzdW6dWuFhIRo+fLluv/++wt9P8MwZLPZnM//+nNhc/5q9OjRGjFihPN5eno6pQgAgBtUmVghGjZsmL7++mutWbNG9erVu+rcwMBAhYSE6ODBg5KkgIAAZWdnKzU11WVeSkqK/P39nXNOnjyZ771OnTrlnHMlLy8vVa9e3eUBAABuTKYWIsMw9Oyzz2rp0qVavXq1QkND//Y1Z86c0bFjxxQYGChJatWqlSpVqqRVq1Y55yQlJWn37t1q166dJCkiIkIOh0Nbtmxxztm8ebMcDodzDgAAsC5TD5kNHTpUn376qb766iv5+Pg4z+ex2+2qUqWKMjIyNHbsWP3zn/9UYGCgjhw5on//+9+qXbu2/vGPfzjnDh48WCNHjlStWrXk6+urUaNGKTw83HnVWdOmTdWrVy8NGTJEH374oSTpiSeeUJ8+fbjCDAAAmFuIZs6cKUnq1KmTy/icOXP06KOPysPDQ7t27dL8+fOVlpamwMBAde7cWYsXL5aPj49z/tSpU1WxYkX1799fFy5cUNeuXTV37lx5eHg45yxcuFDDhw93Xo3Wr18/zZgx4/rvJAAAKPNshmEYZocoD9LT02W32+VwOK7pfKIGLy8vwVTX7sjEe8yOAADAdVPUv99l4qRqAAAAM1GIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5ZlaiCZMmKA77rhDPj4+8vPz03333acDBw64zDEMQ2PHjlVQUJCqVKmiTp06ac+ePS5zsrKyNGzYMNWuXVve3t7q16+fjh8/7jInNTVVkZGRstvtstvtioyMVFpa2vXeRQAAUA6YWojWrVunoUOHKi4uTqtWrdKlS5fUo0cPZWZmOudMmjRJU6ZM0YwZM7R161YFBASoe/fuOnfunHNOdHS0li1bpkWLFmn9+vXKyMhQnz59lJub65wzcOBAxcfHKzY2VrGxsYqPj1dkZGSp7i8AACibbIZhGGaHuOzUqVPy8/PTunXr1KFDBxmGoaCgIEVHR+ull16S9OdqkL+/v95++209+eSTcjgcqlOnjmJiYjRgwABJ0h9//KHg4GCtWLFCPXv21L59+9SsWTPFxcWpTZs2kqS4uDhFRERo//79CgsL+9ts6enpstvtcjgcql69utv72ODl5W6/9no4MvEesyMAAHDdFPXvd5k6h8jhcEiSfH19JUmJiYlKTk5Wjx49nHO8vLzUsWNHbdy4UZK0fft25eTkuMwJCgpS8+bNnXM2bdoku93uLEOS1LZtW9ntduecK2VlZSk9Pd3lAQAAbkxlphAZhqERI0borrvuUvPmzSVJycnJkiR/f3+Xuf7+/s5tycnJ8vT0VM2aNa86x8/PL9/v9PPzc8650oQJE5znG9ntdgUHB1/bDgIAgDKrzBSiZ599Vjt37tRnn32Wb5vNZnN5bhhGvrErXTmnoPlXe5/Ro0fL4XA4H8eOHSvKbgAAgHKoTBSiYcOG6euvv9aaNWtUr14953hAQIAk5VvFSUlJca4aBQQEKDs7W6mpqVedc/LkyXy/99SpU/lWny7z8vJS9erVXR4AAODGZGohMgxDzz77rJYuXarVq1crNDTUZXtoaKgCAgK0atUq51h2drbWrVundu3aSZJatWqlSpUqucxJSkrS7t27nXMiIiLkcDi0ZcsW55zNmzfL4XA45wAAAOuqWBJvkpubq127dikkJCTfuTxXM3ToUH366af66quv5OPj41wJstvtqlKlimw2m6KjozV+/Hg1btxYjRs31vjx41W1alUNHDjQOXfw4MEaOXKkatWqJV9fX40aNUrh4eHq1q2bJKlp06bq1auXhgwZog8//FCS9MQTT6hPnz5FusIMAADc2NxaIYqOjtYnn3wi6c8y1LFjR91+++0KDg7W2rVri/w+M2fOlMPhUKdOnRQYGOh8LF682DnnxRdfVHR0tJ555hm1bt1aJ06c0MqVK+Xj4+OcM3XqVN13333q37+/2rdvr6pVq+qbb76Rh4eHc87ChQsVHh6uHj16qEePHmrRooViYmLc2X0AAHCDces+RPXq1dOXX36p1q1b68svv9TQoUO1Zs0azZ8/X2vWrNGGDRuuR1ZTcR8iAADKn+t6H6LTp087T3hesWKFHnjgAd18880aPHiwdu3a5V5iAAAAk7hViPz9/bV3717l5uYqNjbWea7O+fPnXQ5TAQAAlAdunVQ9aNAg9e/fX4GBgbLZbOrevbukP6/catKkSYkGBAAAuN7cKkRjx45V8+bNdezYMT3wwAPy8vKSJHl4eOjll18u0YAAAADXm9uX3f/P//yPJOnixYvOsaioqGtPBAAAUMrcOocoNzdXb775purWratq1arp8OHDkqQxY8Y4L8cHAAAoL9wqRG+99Zbmzp2rSZMmydPT0zkeHh6ujz/+uMTCAQAAlAa3CtH8+fP10Ucf6aGHHnK5qqxFixbav39/iYUDAAAoDW4VohMnTqhRo0b5xvPy8pSTk3PNoQAAAEqTW4Xolltu0c8//5xvfMmSJWrZsuU1hwIAAChNbl1l9tprrykyMlInTpxQXl6eli5dqgMHDmj+/Pn69ttvSzojAADAdeXWClHfvn21ePFirVixQjabTa+++qr27dunb775xnmTRgAAgPLC7fsQ9ezZUz179izJLAAAAKZwa4Vo69at2rx5c77xzZs3a9u2bdccCgAAoDS5VYiGDh2qY8eO5Rs/ceKEhg4des2hAAAASpNbhWjv3r26/fbb8423bNlSe/fuveZQAAAApcmtQuTl5aWTJ0/mG09KSlLFim6flgQAAGAKtwpR9+7dNXr0aDkcDudYWlqa/v3vf3OVGQAAKHfcWs6ZPHmyOnTooJCQEOeNGOPj4+Xv76+YmJgSDQgAAHC9uVWI6tatq507d2rhwoX69ddfVaVKFQ0aNEj/+te/VKlSpZLOCAAAcF25fcKPt7e3nnjiiZLMAgAAYAq3C9Fvv/2mtWvXKiUlRXl5eS7bXn311WsOBgAAUFrcKkSzZs3S008/rdq1aysgIEA2m8257fJXeQAAAJQXbhWicePG6a233tJLL71U0nkAAABKnVuX3aempuqBBx4o6SwAAACmcKsQPfDAA1q5cmVJZwEAADCFW4fMGjVqpDFjxiguLk7h4eH5LrUfPnx4iYQDAAAoDTbDMIzivig0NLTwN7TZdPjw4WsKVRalp6fLbrfL4XCoevXqbr9Pg5eXl2Cqa3dk4j1mRwAA4Lop6t9vt1aIEhMT3Q4GAABQ1rh1DtFl2dnZOnDggC5dulRSeQAAAEqdW4Xo/PnzGjx4sKpWrapbbrlFR48elfTnuUMTJ04s0YAAAADXm1uFaPTo0fr111+1du1aVa5c2TnerVs3LV68uMTCAQAAlAa3ziH68ssvtXjxYrVt29blLtXNmjVTQkJCiYUDAAAoDW6tEJ06dUp+fn75xjMzM10KEgAAQHngViG64447tHz5/7t8/HIJmjVrliIiIkomGQAAQClx65DZhAkT1KtXL+3du1eXLl3S9OnTtWfPHm3atEnr1q0r6YwAAADXlVsrRO3atdPGjRt1/vx53XTTTVq5cqX8/f21adMmtWrVqqQzAgAAXFfFXiHKycnRE088oTFjxmjevHnXIxMAAECpKvYKUaVKlbRs2bLrkQUAAMAUbh0y+8c//qEvv/yyhKMAAACYw+1vu3/zzTe1ceNGtWrVSt7e3i7b+bZ7AABQnrhViD7++GPVqFFD27dv1/bt21222Ww2ChEAAChX+LZ7AABgedf0bfcAAAA3ArdWiB577LGrbp89e7ZbYQAAAMzgViFKTU11eZ6Tk6Pdu3crLS1NXbp0KZFgAAAApcWtQ2bLli1zeXz77bc6fPiwHnzwQbVt27bI7/PTTz+pb9++CgoKks1my3cp/6OPPiqbzebyuPL9s7KyNGzYMNWuXVve3t7q16+fjh8/7jInNTVVkZGRstvtstvtioyMVFpamju7DgAAbkAldg5RhQoV9Pzzz2vq1KlFfk1mZqZuvfVWzZgxo9A5vXr1UlJSkvOxYsUKl+3R0dFatmyZFi1apPXr1ysjI0N9+vRRbm6uc87AgQMVHx+v2NhYxcbGKj4+XpGRkcXfSQAAcENy65BZYRISEnTp0qUiz+/du7d69+591TleXl4KCAgocJvD4dAnn3yimJgYdevWTZK0YMECBQcH64cfflDPnj21b98+xcbGKi4uTm3atJEkzZo1SxERETpw4IDCwsKKnBcAANyY3CpEI0aMcHluGIaSkpK0fPlyRUVFlUiwy9auXSs/Pz/VqFFDHTt21FtvvSU/Pz9J0vbt25WTk6MePXo45wcFBal58+bauHGjevbsqU2bNslutzvLkCS1bdtWdrtdGzduLLQQZWVlKSsry/k8PT29RPcLAACUHW4Voh07drg8r1ChgurUqaPJkyf/7RVoxdG7d2898MADCgkJUWJiosaMGaMuXbpo+/bt8vLyUnJysjw9PVWzZk2X1/n7+ys5OVmSlJyc7CxQf+Xn5+ecU5AJEybo9ddfL7F9AQAAZZdbhWjNmjUlnaNAAwYMcP7cvHlztW7dWiEhIVq+fLnuv//+Ql9nGIZsNpvz+V9/LmzOlUaPHu2yEpaenq7g4ODi7gIAACgH3DqpOjExUQcPHsw3fvDgQR05cuRaMxUqMDBQISEhzt8dEBCg7OzsfLcBSElJkb+/v3POyZMn873XqVOnnHMK4uXlperVq7s8AADAjcmtQvToo49q48aN+cY3b96sRx999FozFerMmTM6duyYAgMDJUmtWrVSpUqVtGrVKuecpKQk7d69W+3atZMkRUREyOFwaMuWLS45HQ6Hcw4AALA2t88hat++fb7xtm3b6tlnny3y+2RkZOjQoUPO54mJiYqPj5evr698fX01duxY/fOf/1RgYKCOHDmif//736pdu7b+8Y9/SJLsdrsGDx6skSNHqlatWvL19dWoUaMUHh7uvOqsadOm6tWrl4YMGaIPP/xQkvTEE0+oT58+XGEGAAAkuVmIbDabzp07l2/c4XC43P/n72zbtk2dO3d2Pr98zk5UVJRmzpypXbt2af78+UpLS1NgYKA6d+6sxYsXy8fHx/maqVOnqmLFiurfv78uXLigrl27au7cufLw8HDOWbhwoYYPH+68Gq1fv35XvfcRAACwFpthGEZxX9SnTx9VrVpVn332mbN45ObmasCAAcrMzNR3331X4kHNlp6eLrvdLofDcU3nEzV4eXkJprp2RybeY3YEAACum6L+/XZrhWjSpEnq0KGDwsLCdPfdd0uSfv75Z6Wnp2v16tXuJQYAADCJWydVN2vWTDt37lT//v2VkpKic+fO6ZFHHtH+/fvVvHnzks4IAABwXbn91R1BQUEaP358SWYBAAAwhVsrRHPmzNGSJUvyjS9ZskTz5s275lAAAAClya1CNHHiRNWuXTvfuJ+fH6tGAACg3HGrEP3+++8KDQ3NNx4SEqKjR49ecygAAIDS5FYh8vPz086dO/ON//rrr6pVq9Y1hwIAAChNbhWiBx98UMOHD9eaNWuUm5ur3NxcrV69Ws8995wefPDBks4IAABwXbl1ldm4ceP0+++/q2vXrqpY8c+3yM3NVVRUFOcQAQCAcsetQuTp6anFixdr1KhRSkxMVNWqVRUeHq6QkJCSzgcAAHDdFbsQpaWl6ZVXXtHixYuVmpoqSapZs6YefPBBjRs3TjVq1CjpjAAAANdVsQrR2bNnFRERoRMnTuihhx5S06ZNZRiG9u3bp7lz5+rHH3/Uxo0bVbNmzeuVFwAAoMQVqxC98cYb8vT0VEJCgvz9/fNt69Gjh9544w1NnTq1REMCAABcT8W6yuzLL7/Uu+++m68MSVJAQIAmTZqkZcuWlVg4AACA0lCsQpSUlKRbbrml0O3NmzdXcnLyNYcCAAAoTcUqRLVr19aRI0cK3Z6YmMiNGQEAQLlTrELUq1cvvfLKK8rOzs63LSsrS2PGjFGvXr1KLBwAAEBpKNZJ1a+//rpat26txo0ba+jQoWrSpIkkae/evXr//feVlZWlmJiY6xIUAADgeilWIapXr542bdqkZ555RqNHj5ZhGJIkm82m7t27a8aMGQoODr4uQQEAAK6XYt+YMTQ0VN99951SU1N18OBBSVKjRo3k6+tb4uEAAABKg1tf3SH9eXfqO++8sySzAAAAmMKtb7sHAAC4kVCIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VGIAACA5VU0OwBwWYOXl5sdwcWRifeYHQEAUEpYIQIAAJZnaiH66aef1LdvXwUFBclms+nLL7902W4YhsaOHaugoCBVqVJFnTp10p49e1zmZGVladiwYapdu7a8vb3Vr18/HT9+3GVOamqqIiMjZbfbZbfbFRkZqbS0tOu8dwAAoLwwtRBlZmbq1ltv1YwZMwrcPmnSJE2ZMkUzZszQ1q1bFRAQoO7du+vcuXPOOdHR0Vq2bJkWLVqk9evXKyMjQ3369FFubq5zzsCBAxUfH6/Y2FjFxsYqPj5ekZGR133/AABA+WDqOUS9e/dW7969C9xmGIamTZumV155Rffff78kad68efL399enn36qJ598Ug6HQ5988oliYmLUrVs3SdKCBQsUHBysH374QT179tS+ffsUGxuruLg4tWnTRpI0a9YsRURE6MCBAwoLCyudnQUAAGVWmT2HKDExUcnJyerRo4dzzMvLSx07dtTGjRslSdu3b1dOTo7LnKCgIDVv3tw5Z9OmTbLb7c4yJElt27aV3W53zilIVlaW0tPTXR4AAODGVGYLUXJysiTJ39/fZdzf39+5LTk5WZ6enqpZs+ZV5/j5+eV7fz8/P+ecgkyYMMF5zpHdbldwcPA17Q8AACi7ymwhusxms7k8Nwwj39iVrpxT0Py/e5/Ro0fL4XA4H8eOHStmcgAAUF6U2UIUEBAgSflWcVJSUpyrRgEBAcrOzlZqaupV55w8eTLf+586dSrf6tNfeXl5qXr16i4PAABwYyqzhSg0NFQBAQFatWqVcyw7O1vr1q1Tu3btJEmtWrVSpUqVXOYkJSVp9+7dzjkRERFyOBzasmWLc87mzZvlcDiccwAAgLWZepVZRkaGDh065HyemJio+Ph4+fr6qn79+oqOjtb48ePVuHFjNW7cWOPHj1fVqlU1cOBASZLdbtfgwYM1cuRI1apVS76+vho1apTCw8OdV501bdpUvXr10pAhQ/Thhx9Kkp544gn16dOHK8wAAIAkkwvRtm3b1LlzZ+fzESNGSJKioqI0d+5cvfjii7pw4YKeeeYZpaamqk2bNlq5cqV8fHycr5k6daoqVqyo/v3768KFC+ratavmzp0rDw8P55yFCxdq+PDhzqvR+vXrV+i9jwAAgPXYDMMwzA5RHqSnp8tut8vhcFzT+UR8X1fh+GwAACWtqH+/y+w5RAAAAKWFQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgQAACyvTBeisWPHymazuTwCAgKc2w3D0NixYxUUFKQqVaqoU6dO2rNnj8t7ZGVladiwYapdu7a8vb3Vr18/HT9+vLR3BQAAlGFluhBJ0i233KKkpCTnY9euXc5tkyZN0pQpUzRjxgxt3bpVAQEB6t69u86dO+ecEx0drWXLlmnRokVav369MjIy1KdPH+Xm5pqxOwAAoAyqaHaAv1OxYkWXVaHLDMPQtGnT9Morr+j++++XJM2bN0/+/v769NNP9eSTT8rhcOiTTz5RTEyMunXrJklasGCBgoOD9cMPP6hnz56lui8AAKBsKvMrRAcPHlRQUJBCQ0P14IMP6vDhw5KkxMREJScnq0ePHs65Xl5e6tixozZu3ChJ2r59u3JyclzmBAUFqXnz5s45AAAAZXqFqE2bNpo/f75uvvlmnTx5UuPGjVO7du20Z88eJScnS5L8/f1dXuPv76/ff/9dkpScnCxPT0/VrFkz35zLry9MVlaWsrKynM/T09NLYpcAAEAZVKYLUe/evZ0/h4eHKyIiQjfddJPmzZuntm3bSpJsNpvLawzDyDd2paLMmTBhgl5//XU3kwMAgPKkzB8y+ytvb2+Fh4fr4MGDzvOKrlzpSUlJca4aBQQEKDs7W6mpqYXOKczo0aPlcDicj2PHjpXgngAAgLKkXBWirKws7du3T4GBgQoNDVVAQIBWrVrl3J6dna1169apXbt2kqRWrVqpUqVKLnOSkpK0e/du55zCeHl5qXr16i4PAABwYyrTh8xGjRqlvn37qn79+kpJSdG4ceOUnp6uqKgo2Ww2RUdHa/z48WrcuLEaN26s8ePHq2rVqho4cKAkyW63a/DgwRo5cqRq1aolX19fjRo1SuHh4c6rzgAAAMp0ITp+/Lj+9a9/6fTp06pTp47atm2ruLg4hYSESJJefPFFXbhwQc8884xSU1PVpk0brVy5Uj4+Ps73mDp1qipWrKj+/fvrwoUL6tq1q+bOnSsPDw+zdgsAAJQxNsMwDLNDlAfp6emy2+1yOBzXdPiswcvLSzDVtTsy8R6zIzjx2QAASlpR/36Xq3OIAAAArgcKEQAAsDwKEQAAsDwKEQAAsDwKEQAAsLwyfdk9gD9xBR4AXF+sEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMujEAEAAMuraHYAALgWDV5ebnYEpyMT7zE7AgA3sUIEAAAsjxUiALhBsXoGFB0rRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIoRAAAwPIsVYjef/99hYaGqnLlymrVqpV+/vlnsyMBAIAywDKFaPHixYqOjtYrr7yiHTt26O6771bv3r119OhRs6MBAACTWaYQTZkyRYMHD9bjjz+upk2batq0aQoODtbMmTPNjgYAAExW0ewApSE7O1vbt2/Xyy+/7DLeo0cPbdy4scDXZGVlKSsry/nc4XBIktLT068pS17W+Wt6fUm71v0pSXw2heOzKVxZ+mzK0uci8dkA0v/7Z88wjKvOs0QhOn36tHJzc+Xv7+8y7u/vr+Tk5AJfM2HCBL3++uv5xoODg69LRrPYp5mdoOzisykcn03B+FwKx2cDs507d052u73Q7ZYoRJfZbDaX54Zh5Bu7bPTo0RoxYoTzeV5ens6ePatatWoV+prSkp6eruDgYB07dkzVq1c3NUtZw2dTOD6bwvHZFI7PpmB8LoUra5+NYRg6d+6cgoKCrjrPEoWodu3a8vDwyLcalJKSkm/V6DIvLy95eXm5jNWoUeN6RXRL9erVy8Q/bGURn03h+GwKx2dTOD6bgvG5FK4sfTZXWxm6zBInVXt6eqpVq1ZatWqVy/iqVavUrl07k1IBAICywhIrRJI0YsQIRUZGqnXr1oqIiNBHH32ko0eP6qmnnjI7GgAAMJllCtGAAQN05swZvfHGG0pKSlLz5s21YsUKhYSEmB2t2Ly8vPTaa6/lO6QHPpur4bMpHJ9N4fhsCsbnUrjy+tnYjL+7Dg0AAOAGZ4lziAAAAK6GQgQAACyPQgQAACyPQgQAACyPQgQAACyPQgTAkh577DGdO3cu33hmZqYee+wxExIB5dfPP/+shx9+WBERETpx4oQkKSYmRuvXrzc5WdFRiMqZ8+fPa//+/dq5c6fLA0DxzJs3TxcuXMg3fuHCBc2fP9+ERED59MUXX6hnz56qUqWKduzYoaysLEl/fpnq+PHjTU5XdJa5MWN5d+rUKQ0aNEjfffddgdtzc3NLOVHZdOHCBeXk5LiMlZXv0jHLb7/9prVr1yolJUV5eXku21599VWTUpknPT1dhmE4v/CxcuXKzm25ublasWKF/Pz8TExovv/85z8FjttsNlWuXFmNGjVShw4d5OHhUcrJyoaEhATNmTNHCQkJmj59uvz8/BQbG6vg4GDdcsstZscrdePGjdMHH3ygRx55RIsWLXKOt2vXTm+88YaJyYqHQlROREdHKzU1VXFxcercubOWLVumkydPaty4cZo8ebLZ8Ux1/vx5vfjii/r888915syZfNutXBZnzZqlp59+WrVr11ZAQIBsNptzm81ms2QhqlGjhmw2m2w2m26++eZ82202m15//XUTkpUdU6dO1alTp3T+/HnVrFlThmEoLS1NVatWVbVq1ZSSkqKGDRtqzZo1Cg4ONjtuqVq3bp169+6t9u3b66efftJbb70lPz8/7dy5Ux9//LH++9//mh2x1B04cEAdOnTIN169enWlpaWVfiB3GSgXAgICjM2bNxuGYRg+Pj7GgQMHDMMwjK+++spo3769mdFM98wzzxhNmzY1lixZYlSpUsWYPXu28eabbxr16tUzFixYYHY8U9WvX9+YOHGi2THKlLVr1xpr1qwxbDabsXTpUmPt2rXOx8aNG40TJ06YHdF0n376qdGpUyfj0KFDzrGDBw8aXbp0MRYtWmQcO3bMaN++vfHPf/7TxJTmaNu2rTF58mTDMAyjWrVqRkJCgmEYhrFlyxYjKCjIzGimadiwobFq1SrDMFw/k3nz5hlNmzY1M1qxUIjKCR8fHyMxMdEwDMMICQkx1q9fbxiGYRw+fNioUqWKicnMFxwcbKxZs8YwjD8/p4MHDxqGYRjz5883evfubWIy8/n4+Dj/5QRXR44cMXJzc82OUSY1bNjQ2LFjR77xX375xQgNDTUMwzA2bNhgBAQElHIy83l7exuHDx82DMP1j39iYqLh5eVlZjTTvP3220azZs2MuLg4w8fHx/j555+NBQsWGHXq1DHee+89s+MVGYfMyomwsDAdOHBADRo00G233aYPP/xQDRo00AcffKDAwECz45nq7NmzCg0NlfTnEu3Zs2clSXfddZeefvppM6OZ7oEHHtDKlSv11FNPmR2lzLn8xc7nz5/X0aNHlZ2d7bK9RYsWZsQqE5KSknTp0qV845cuXVJycrIkKSgoqMCr9G50NWrUUFJSkvPfOZft2LFDdevWNSmVuV588UU5HA517txZFy9eVIcOHeTl5aVRo0bp2WefNTtekVGIyono6GglJSVJkl577TX17NlTCxYskKenp+bNm2dyOnM1bNhQR44cUUhIiJo1a6bPP/9cd955p7755hvVqFHD7HimatSokcaMGaO4uDiFh4erUqVKLtuHDx9uUjLzcaFC4Tp37qwnn3xSH3/8sVq2bCnpzz/4Tz/9tLp06SJJ2rVrV75SYAUDBw7USy+9pCVLlshmsykvL08bNmzQqFGj9Mgjj5gdzzRvvfWWXnnlFe3du1d5eXlq1qyZqlWrZnas4jF7iQruyczMNLZv326cOnXK7CimmzJlijF9+nTDMAxj9erVRpUqVQxPT0+jQoUKxrRp00xOZ64GDRoU+rh86MOqBg4caLRr187YsmWL4e3tbaxcudKIiYkxwsLCjG+//dbseKZKSkoyunXrZthsNsPT09Pw9PQ0bDab0b17dyMpKckwjD//v/b999+bnLT0ZWdnGwMHDjQqVKhg2Gw2o1KlSkaFChWMhx9+2Lh06ZLZ8UwxaNAgIz09Pd94RkaGMWjQIBMSucdmGIZhdinD3xsxYkSB43+9DPbee++Vr69vKScre44ePapt27bppptu0q233mp2HJRRgYGB+uqrr3TnnXeqevXq2rZtm26++WZ9/fXXmjRpUrm6odz1cuDAAR04cECGYahJkyYKCwszO1KZkZCQoB07digvL08tW7ZU48aNzY5kGg8PDyUlJeW7XcXp06cVEBBQ4OHXsohDZuXEjh079Msvvyg3N1dhYWEyDEMHDx6Uh4eHmjRpovfff18jR47U+vXr1axZM7Pjlroff/xRP/74Y4H32pk9e7ZJqcwxYsQIvfnmm/L29i60SEt/lmkr37IhMzPT+S9wX19fnTp1SjfffLPCw8P1yy+/mJyu9P3dPzfr1q1z/jxlypTSjFYm3XTTTbrpppvMjmGqG+2eXhSicuLy6s+cOXOcNxpMT0/X4MGDddddd2nIkCEaOHCgnn/+eX3//fcmpy1dr7/+ut544w21bt1agYGBLvfasaIdO3Y4b065Y8eOQudZ/XPiQgVX/HNTuKv9h8WVrFQWb7R7enHIrJyoW7euVq1alW/1Z8+ePerRo4dOnDihX375RT169NDp06dNSmmOwMBATZo0SZGRkWZHQTmycOFC5eTk6NFHH9WOHTvUs2dPnT592nmhwoABA8yOiDKic+fOLs+3b9/uXK2X/rwbvIeHh1q1aqXVq1ebEdEU69atk2EY6tKli7744guXUzY8PT0VEhKioKAgExMWDytE5YTD4VBKSkq+QnTq1Cmlp6dL+rOtX3npsBVkZ2erXbt2ZsdAOfPQQw85f27ZsqWOHDmi/fv3q379+qpdu7aJyVDWrFmzxvnzlClT5OPjo3nz5qlmzZqSpNTUVA0aNEh33323WRFN0bFjR0lSYmKi6tevX+Dq4dGjR1W/fv3SjuYWVojKiYceekibNm3S5MmTdccdd8hms2nLli0aNWqU2rVrp5iYGC1atEjvvvuutm3bZnbcUvXSSy+pWrVqGjNmjNlRUI5woQLcUbduXa1cuTLfd5bt3r1bPXr00B9//GFSMvMUdlL1mTNn5OfnV25uYcEKUTnx4Ycf6vnnn9eDDz7oPGO/YsWKioqK0tSpUyVJTZo00ccff2xmTFNcvHhRH330kX744Qe1aNEi3712rHRMH0XHhQpwR3p6uk6ePJmvEKWkpFjyRpWSVNi6SkZGhsuJ1mUdK0TlTEZGhg4fPizDMHTTTTeVvxtfXQdXHt//K5vNZqlj+ii6adOm6eeff/7bCxUuXLhguQsVULhHHnlE69at0+TJk9W2bVtJUlxcnF544QV16NDBUjfKvbzKOn36dA0ZMkRVq1Z1bsvNzdXmzZvl4eGhDRs2mBWxWChEACyJCxXgjvPnz2vUqFGaPXu286q8ihUravDgwXrnnXfk7e1tcsLSc/k/RtetW6eIiAh5eno6t3l6eqpBgwYaNWpUublHE4UIgCVVq1ZN3377rTp16uQyvnbtWvXt21fnzp3T4cOHddtttzkvXAAuy8zMVEJCggzDUKNGjSxVhK40aNAgTZ8+3bnSWl5xDhEAS7r33nv12GOPFXihwn333SdJ2rJlS4H3VwG8vb0t/QXAfzVnzhxJ0qFDh5SQkKAOHTqoSpUqMgyjXN23ihUiAJaUkZGh559/XvPnzy/wQgVvb2/Fx8dLkm677TbzgqLM2bp1q5YsWaKjR4/mu9XJ0qVLTUplnrNnz+qBBx7QmjVrZLPZdPDgQTVs2FCDBw9WjRo1ys0d8SuYHQAAzFCtWjXNmjVLZ86ccV5xdubMGX300UfOwx+33XYbZQguFi1apPbt22vv3r1atmyZcnJytHfvXq1evVp2u93seKaIjo5WpUqVdPToUZcTqwcMGKDY2FgTkxUPh8wAWFq1atU49IEiGz9+vKZOnaqhQ4fKx8dH06dPV2hoqJ588klLfuWLJK1cuVLff/+96tWr5zLeuHFj/f777yalKj5WiAAAKKKEhATdc889kiQvLy9lZmbKZrPp+eef10cffWRyOnNkZma6rAxddvr0aXl5eZmQyD0UIgAAisjX19d5A8a6detq9+7dkqS0tDSdP3/ezGim6dChg+bPn+98brPZlJeXp3feeeeq94krazhkBgBAEd19991atWqVwsPD1b9/fz333HNavXq1Vq1apa5du5odzxTvvPOOOnXqpG3btik7O1svvvii9uzZo7Nnz5abmzJKXGUGAECRnT17VhcvXlRQUJDy8vL07rvvav369WrUqJHGjBnj/MJXq0lOTtbMmTO1fft25eXl6fbbb9fQoUPL1XlVFCIAAGB5HDIDAKAY8vLydOjQIaWkpCgvL89lW4cOHUxKVbp27txZ5Lnl5SpOVogAACiiuLg4DRw4UL///nu+b3m32WzKzc01KVnpqlChgmw2W6HfdH9ZefpMWCECAKCInnrqKbVu3VrLly9XYGBgufpqipKUmJhodoQSxwoRAABF5O3trV9//VWNGjUyOwpKGPchAgCgiNq0aaNDhw6ZHaNMmTBhgmbPnp1vfPbs2Xr77bdNSOQeVogAACiiZcuW6X//93/1wgsvKDw8XJUqVXLZXl5OIC5JDRo00Keffqp27dq5jG/evFkPPvhguTm8RiECAKCIKlTIf2Dl8snF5ekE4pJUuXJl7du3T6GhoS7jhw8fVrNmzXTx4kWTkhUPJ1UDAFBE5WW1ozQFBwdrw4YN+QrRhg0bFBQUZFKq4qMQAQBQRCEhIWZHKHMef/xxRUdHKycnR126dJEk/fjjj3rxxRc1cuRIk9MVHYfMAAAohoSEBE2bNk379u2TzWZT06ZN9dxzz+mmm24yO5opDMPQyy+/rP/85z/Kzs6W9OdhtJdeekmvvvqqyemKjkIEAEARff/99+rXr59uu+02tW/fXoZhaOPGjfr111/1zTffqHv37mZHNE1GRob27dunKlWqqHHjxvLy8jI7UrFQiAAAKKKWLVuqZ8+emjhxosv4yy+/rJUrV+qXX34xKZl55s6dqwEDBqhKlSpmR7kmFCIAAIqocuXK2rVrlxo3buwy/ttvv6lFixbl5oqqkhQYGKjMzEw98MADGjx4cL7L78sLbswIAEAR1alTR/Hx8fnG4+Pj5efnV/qByoDjx49rwYIFSk1NVefOndWkSRO9/fbbSk5ONjtasXCVGQAARTRkyBA98cQTOnz4sNq1ayebzab169fr7bffLldXVJUkDw8P9evXT/369VNKSooWLFiguXPnasyYMerVq5cGDx6svn37FngPp7KEQ2YAABSRYRiaNm2aJk+erD/++EOSFBQUpBdeeEHDhw+37Je9/tXmzZs1e/ZszZs3T4GBgUpLS1ONGjU0Z84cderUyex4haIQAQDghnPnzkmSfHx8TE5ivpMnTyomJkZz5szR4cOHdd9992nw4MHq1q2bLly4oP/93//Vf//7X/3+++9mRy0UhQgAALitb9+++v7773XzzTfr8ccf1yOPPCJfX1+XOX/88Yfq1aunvLw8k1L+Pc4hAgCgiM6cOaNXX31Va9asUUpKSr4/8GfPnjUpmXn8/Py0bt06RUREFDonMDCwzH/tCStEAAAUUe/evZWQkKDBgwfL398/3zlDUVFRJiXDtaIQAQBQRD4+Plq/fr1uvfVWs6OUGcOHD1ejRo00fPhwl/EZM2bo0KFDmjZtmjnBiqlsXwMHAEAZ0qRJE124cMHsGGXKF198ofbt2+cbb9eunf773/+akMg9FCIAAIro/fff1yuvvKJ169bpzJkzSk9Pd3lY0ZkzZ2S32/ONV69eXadPnzYhkXsoRAAAFFGNGjXkcDjUpUsX+fn5qWbNmqpZs6Zq1KihmjVrmh3PFI0aNVJsbGy+8e+++04NGzY0IZF7uMoMAIAieuihh+Tp6alPP/20wJOqrWjEiBF69tlnderUKXXp0kWS9OOPP2ry5Mnl5vwhiZOqAQAosqpVq2rHjh0KCwszO0qZMnPmTL311lvOu3c3aNBAY8eO1SOPPGJysqKjEAEAUEQdOnTQq6++qm7dupkdpUw6deqUqlSpomrVqpkdpdgoRAAAFNGSJUs0duxYvfDCCwoPD1elSpVctrdo0cKkZLhWFCIAAIqooG9st9lsMgxDNptNubm5JqQqfbfffrt+/PFH1axZUy1btrzquVS//PJLKSZzHydVAwBQRGX96ydKy7333isvLy9J0n333WdumBLCChEAALA87kMEAEAxxMTEqH379goKCtLvv/8uSZo2bZq++uork5OZa9u2bYqJidGCBQu0fft2s+MUG4UIAIAimjlzpkaMGKH/7//7/5SWluY8Z6hGjRrl6p47Jen48eO6++67deedd+q5557T8OHDdccdd+iuu+7SsWPHzI5XZBQiAACK6L333tOsWbP0yiuvyMPDwzneunVr7dq1y8Rk5nnssceUk5Ojffv26ezZszp79qz27dsnwzA0ePBgs+MVGSdVAwBQRImJiWrZsmW+cS8vL2VmZpqQyHw///yzNm7c6HKzyrCwML333nsFfulrWcUKEQAARRQaGqr4+Ph84999952aNWtW+oHKgPr16ysnJyff+KVLl1S3bl0TErmHFSIAAIrohRde0NChQ3Xx4kUZhqEtW7bos88+04QJE/Txxx+bHc8UkyZN0rBhw/R///d/atWqlWw2m7Zt26bnnntO7777rtnxiozL7gEAKIZZs2Zp3LhxzhOG69atq7Fjx5ar82WuVc2aNV1uxpiZmalLly6pYsU/11ku/+zt7a2zZ8+aFbNYKEQAALjh9OnTysvLk5+fnzIzM7V9+3Z16NDB7FilYt68eUWeGxUVdR2TlBwKEQAA1+jXX3/V7bffbpmv7rgRcVI1AACwPAoRAACwPAoRAACwPC67BwDgb3z99ddX3Z6YmFhKSXC9cFI1AAB/o0KFvz+gYrPZOKm6HKMQAQCAa7J161YtWbJER48eVXZ2tsu2pUuXmpSqeDiHCACAIjh//nyhX+C6Z88eZWRklHKismHRokVq37699u7dq2XLliknJ0d79+7V6tWrZbfbzY5XZBQiAACKIDs7W23atNGWLVtcxvfu3auWLVtathCNHz9eU6dO1bfffitPT09Nnz5d+/btU//+/VW/fn2z4xUZhQgAgCKoUaOG+vbtm+8uzTExMerWrZsCAgJMSmauhIQE3XPPPZIkLy8vZWZmymaz6fnnn9dHH31kcrqioxABAFBEjzzyiD7//HNdunRJkmQYhhYuXKhHH33U3GAm8vX11blz5yT9+b1uu3fvliSlpaXp/PnzZkYrFgoRAABF1KtXL1WsWFHLly+XJK1Zs0YZGRm67777zA1morvvvlurVq2SJPXv31/PPfechgwZon/961/q2rWryemKjqvMAAAohhdeeEEJCQlaunSpBg0apMqVK2vmzJlmxzLN2bNndfHiRQUFBSkvL0/vvvuu1q9fr0aNGmnMmDGqWbOm2RGLhEIEAEAx7Nq1S3feeacOHTqkpk2bauXKlWrbtq3ZscqczMxMbd++XR06dDA7SpFQiAAAKKZWrVrJx8dHycnJ2r9/v9lxyqRff/1Vt99+e7m5WSXnEAEAUExRUVH66aefFBUVZXYUlBC+ywwAgGJ6+OGHlZqaqkGDBpkdBSWEQ2YAAKDElbdDZqwQAQCAYvv666+vuj0xMbGUkpQMVogAAECxVajw96ch22y2crNCRCECAACWx1VmAADA8ihEAADALZmZmdq1a1eB2/bs2aOMjIxSTuQ+ChEAAHBLTk6O2rRpoy1btriM7927Vy1btqQQAQCAG1+NGjXUt29fzZs3z2U8JiZG3bp1U0BAgEnJio9CBAAA3PbII4/o888/16VLlyRJhmFo4cKFevTRR80NVkwUIgAA4LZevXqpYsWKWr58uSRpzZo1ysjI0H333WdusGKiEAEAALd5eHjo4Ycfdh42i4mJ0YABA+Tp6WlysuLhPkQAAOCa7Nq1S3feeacOHTqkpk2bauXKlWrbtq3ZsYqFQgQAAK5Zq1at5OPjo+TkZO3fv9/sOMXGITMAAHDNoqKi9NNPPykqKsrsKG7hy10BAMA1e/jhh5WamqpBgwaZHcUtHDIDAACWxyEzAABgeRQiAABgeRQiAABgeRQiAABgeRQiAJbUoUMHffrpp87nycnJ6t69u7y9vVWjRg1Jks1m05dffmlOwBLy7bffqmXLlsrLyzM7ClCmUYgAXFVycrKee+45NWrUSJUrV5a/v7/uuusuffDBBzp//rzZ8dzy7bffKjk5WQ8++KBzbOrUqUpKSlJ8fLx+++03SVJSUpJ69+7t9u9Zu3atbDab0tLSrjpv7ty5zhJW0vr06SObzeZS/gDkx32IABTq8OHDat++vWrUqKHx48crPDxcly5d0m+//abZs2crKChI/fr1K7Hfl5OTo0qVKrmMZWdnu/WdSFd73X/+8x8NGjRIFSr8v/8mTEhIUKtWrdS4cWPnWEBAQLHzlkWDBg3Se++9p4cfftjsKEDZZQBAIXr27GnUq1fPyMjIKHB7Xl7eVV8/e/Zso0mTJoaXl5cRFhZm/N///Z9zW2JioiHJWLx4sdGxY0fDy8vLmD17thEVFWXce++9xvjx443AwEAjJCTEMAzD2Llzp9G5c2ejcuXKhq+vrzFkyBDj3Llzzvcr7HVXOnXqlGGz2Yzdu3c7x0JCQgxJzkdUVJRhGIYhyVi2bNlV8x45csTo06ePUaNGDaNq1apGs2bNjOXLlzvnF/S+f7VmzZp881577TXDMAwjJibGaNWqlVGtWjXD39/f+Ne//mWcPHnS5fVfffWV0ahRI6Ny5cpGp06djLlz5xqSjNTUVOecI0eOGJKMhISEq/7vBVgZhQhAgU6fPm3YbDZjwoQJbr3+o48+MgIDA40vvvjCOHz4sPHFF18Yvr6+xty5cw3D+H8Fo0GDBs45J06cMKKiooxq1aoZkZGRxu7du41du3YZmZmZRlBQkHH//fcbu3btMn788UcjNDTUpWAU9LqCLFu2zPD29jZyc3OdYykpKUavXr2M/v37G0lJSUZaWpphGAUXoivz3nPPPUb37t2NnTt3GgkJCcY333xjrFu3zrh06ZLxxRdfGJKMAwcOuLzvX2VlZRnTpk0zqlevbiQlJRlJSUnOovfJJ58YK1asMBISEoxNmzYZbdu2NXr37u18bWJiolGpUiVj1KhRxv79+43PPvvMqFu3br5CZBiG4efn5/zsAeTHITMABTp06JAMw1BYWJjLeO3atXXx4kVJ0tChQ/X2228X+Po333xTkydP1v333y9JCg0N1d69e/Xhhx+6fNdRdHS0c85l3t7e+vjjj52HvGbNmqULFy5o/vz58vb2liTNmDFDffv21dtvvy1/f/8CX1eQI0eOyN/f3+VwWZ06deTl5aUqVar87WGyK/MePXpU//znPxUeHi5JatiwoXObr6+vJMnPz6/Qc4Q8PT1lt9tls9ny/e7HHnvM+XPDhg31n//8R3feeacyMjJUrVo1ffDBBwoLC9M777wjSQoLC9Pu3bv11ltv5fs9devW1ZEjR666b4CVcVI1gKuy2Wwuz7ds2aL4+HjdcsstysrKKvA1p06d0rFjxzR48GBVq1bN+Rg3bpwSEhJc5rZu3Trf68PDw11Kzb59+3Trrbc6y5AktW/fXnl5eTpw4EChryvIhQsXVLly5avOuZor8w4fPlzjxo1T+/bt9dprr2nnzp1uv/eVduzYoXvvvVchISHy8fFRp06dJP1ZwiTpwIEDuuOOO1xec+eddxb4XlWqVCm3J8EDpYFCBKBAjRo1ks1m0/79+13GGzZsqEaNGqlKlSqFvvbyJd6zZs1SfHy887F7927FxcW5zP1rySlszDCMfMXssr+OF/ReV6pdu7ZSU1P/dl5hrvwdjz/+uA4fPqzIyEjt2rVLrVu31nvvvef2+1+WmZmpHj16qFq1alqwYIG2bt2qZcuWSfrzhHGp4M/FKOTrKc+ePas6depccy7gRkUhAlCgWrVqqXv37poxY4YyMzOL9Vp/f3/VrVtXhw8fVqNGjVweoaGhxc7SrFkzxcfHu+TYsGGDKlSooJtvvrlY79WyZUslJydfUym6UnBwsJ566iktXbpUI0eO1KxZsyTJuVqVm5t71dd7enrmm7N//36dPn1aEydO1N13360mTZooJSXFZU6TJk20detWl7Ft27ble/+LFy8qISFBLVu2LPa+AVZBIQJQqPfff1+XLl1S69attXjxYu3bt08HDhzQggULtH//fnl4eBT62rFjx2rChAmaPn26fvvtN+3atUtz5szRlClTip3joYceUuXKlRUVFaXdu3drzZo1GjZsmCIjI53nDxVVy5YtVadOHW3YsKHYOQoSHR2t77//XomJifrll1+0evVqNW3aVJIUEhIim82mb7/9VqdOnVJGRkaB79GgQQNlZGToxx9/1OnTp3X+/HnVr19fnp6eeu+993T48GF9/fXXevPNN11e9+STT2r//v166aWX9Ntvv+nzzz/X3LlzJbmunMXFxcnLy0sRERElss/AjYhCBKBQN910k3bs2KFu3bpp9OjRuvXWW52HhEaNGpXvD/RfPf744/r44481d+5chYeHq2PHjpo7d65bK0RVq1bV999/r7Nnz+qOO+7Q//zP/6hr166aMWNGsd/Lw8NDjz32mBYuXFjs1xYkNzdXQ4cOVdOmTdWrVy+FhYXp/fffl/Tnicyvv/66Xn75Zfn7++vZZ58t8D3atWunp556SgMGDFCdOnU0adIk1alTR3PnztWSJUvUrFkzTZw4Ue+++67L60JDQ/Xf//5XS5cuVYsWLTRz5ky98sorkiQvLy/nvM8++0wPPfSQqlatWiL7DNyIbEZhB5wB4AZ18uRJ3XLLLdq+fbtCQkLMjlOi3nrrLX3wwQc6duyYpD9PcG/SpIm2bdvmVhkFrILL7gFYjr+/vz755BMdPXq03Bei999/X3fccYdq1aqlDRs26J133nFZiUpMTNT7779PGQL+BitEAFCOPf/881q8eLHOnj2r+vXrKzIyUqNHj1bFivz3LlAcFCIAAGB5nFQNAAAsj0IEAAAsj0IEAAAsj0IEAAAsj0IEAAAsj0IEAAAsj0IEAAAsj0IEAAAsj0IEAAAs7/8Hec7Xk3d0dF8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g_split_tags = df_g_tags_exploded.g_annotator_error_tag.str.split(\":\", expand=True)\n",
    "plt.ylabel(\"Occurences\")\n",
    "plt.xlabel(\"G error (first tag)\")\n",
    "g_split_tags[0].value_counts().plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GPrmRSiMO3MP",
    "outputId": "3ac8c64c-468b-4c86-8f67-b8f459f8e1d2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: xlabel='G error (first two tags)', ylabel='Occurences'>"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHhCAYAAACP2aGaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUpklEQVR4nO3deVhUZf8/8PeIgLKNLMJAIeCGC2So5fpzFyS31Cc1/ZILYZZipFaaqbgkfn0exdIsMxVQe7RM1NTIFTdEE8RdRMWNQNBwkCVAuH9/eHm+joALMjMHz/t1Xee6nPvcc85nxtF5z33uc45KCCFAREREpGA1jF0AERERkbExEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHi1TR2AdVFaWkp/vrrL1hbW0OlUhm7HCIiInoGQgjcu3cPLi4uqFGj4nEgBqJn9Ndff8HV1dXYZRAREVEl3LhxA6+++mqF6xmInpG1tTWAB2+ojY2NkashIiKiZ5GTkwNXV1fpe7wiDETP6OFhMhsbGwYiIiKiauZp0104qZqIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFK+msQt42bhP2f7C27g6v3cVVEJERETPiiNEREREpHhGDURhYWF44403YG1tDUdHR7z99ttITk7W6SOEQGhoKFxcXFC7dm106dIFZ8+e1elTWFiI4OBgODg4wNLSEv369cPNmzd1+mRnZyMgIABqtRpqtRoBAQG4e/euvl8iERERVQNGDUT79+/HuHHjEB8fj127duH+/fvw9fVFXl6e1GfBggVYtGgRli5dij///BMajQY9e/bEvXv3pD4hISGIjo7G+vXrcejQIeTm5qJPnz4oKSmR+gwbNgxJSUmIiYlBTEwMkpKSEBAQYNDXS0RERPKkEkIIYxfxUFZWFhwdHbF//3506tQJQgi4uLggJCQEn3/+OYAHo0FOTk743//9X3zwwQfQarWoW7cu1qxZgyFDhgAA/vrrL7i6umLHjh3w8/PD+fPn0axZM8THx6NNmzYAgPj4eLRr1w4XLlyAp6fnU2vLycmBWq2GVquFjY1Nhf04h4iIiEg+nvX7W1ZziLRaLQDAzs4OAJCamoqMjAz4+vpKfczNzdG5c2fExcUBABISElBcXKzTx8XFBV5eXlKfI0eOQK1WS2EIANq2bQu1Wi31eVxhYSFycnJ0FiIiIno5ySYQCSEwceJEdOzYEV5eXgCAjIwMAICTk5NOXycnJ2ldRkYGzMzMYGtr+8Q+jo6OZfbp6Ogo9XlcWFiYNN9IrVbD1dX1xV4gERERyZZsAtH48eNx6tQp/Pe//y2zTqVS6TwWQpRpe9zjfcrr/6TtTJ06FVqtVlpu3LjxLC+DiIiIqiFZBKLg4GBs3boV+/btw6uvviq1azQaACgzipOZmSmNGmk0GhQVFSE7O/uJfW7dulVmv1lZWWVGnx4yNzeHjY2NzkJEREQvJ6MGIiEExo8fj02bNmHv3r3w8PDQWe/h4QGNRoNdu3ZJbUVFRdi/fz/at28PAGjVqhVMTU11+qSnp+PMmTNSn3bt2kGr1eLYsWNSn6NHj0Kr1Up9iIiISLmMeqXqcePG4aeffsKWLVtgbW0tjQSp1WrUrl0bKpUKISEhmDdvHho1aoRGjRph3rx5sLCwwLBhw6S+gYGBmDRpEuzt7WFnZ4fJkyfD29sbPXr0AAA0bdoUvXr1QlBQEJYvXw4AGDNmDPr06fNMZ5gRERHRy82ogei7774DAHTp0kWnffXq1Rg5ciQA4LPPPkNBQQE++ugjZGdno02bNti5cyesra2l/uHh4ahZsyYGDx6MgoICdO/eHRERETAxMZH6rFu3DhMmTJDORuvXrx+WLl2q3xdIRERE1YKsrkMkZ7wOERERUfVTLa9DRERERGQMDERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeEYNRAcOHEDfvn3h4uIClUqFzZs366xXqVTlLv/+97+lPl26dCmzfujQoTrbyc7ORkBAANRqNdRqNQICAnD37l0DvEIiIiKqDowaiPLy8tCiRQssXbq03PXp6ek6y6pVq6BSqTBo0CCdfkFBQTr9li9frrN+2LBhSEpKQkxMDGJiYpCUlISAgAC9vS4iIiKqXmoac+f+/v7w9/evcL1Go9F5vGXLFnTt2hX169fXabewsCjT96Hz588jJiYG8fHxaNOmDQBgxYoVaNeuHZKTk+Hp6fmCr4KIiIiqu2ozh+jWrVvYvn07AgMDy6xbt24dHBwc0Lx5c0yePBn37t2T1h05cgRqtVoKQwDQtm1bqNVqxMXFVbi/wsJC5OTk6CxERET0cjLqCNHziIyMhLW1NQYOHKjTPnz4cHh4eECj0eDMmTOYOnUqTp48iV27dgEAMjIy4OjoWGZ7jo6OyMjIqHB/YWFhmDVrVtW+CCIiIpKlahOIVq1aheHDh6NWrVo67UFBQdKfvby80KhRI7Ru3RqJiYlo2bIlgAeTsx8nhCi3/aGpU6di4sSJ0uOcnBy4urq+6MsgIiIiGaoWgejgwYNITk7Ghg0bntq3ZcuWMDU1RUpKClq2bAmNRoNbt26V6ZeVlQUnJ6cKt2Nubg5zc/MXqpuIiIiqh2oxh2jlypVo1aoVWrRo8dS+Z8+eRXFxMZydnQEA7dq1g1arxbFjx6Q+R48ehVarRfv27fVWMxEREVUfRh0hys3NxaVLl6THqampSEpKgp2dHerVqwfgwaGqX375BQsXLizz/MuXL2PdunV466234ODggHPnzmHSpEnw8fFBhw4dAABNmzZFr169EBQUJJ2OP2bMGPTp04dnmBEREREAI48QHT9+HD4+PvDx8QEATJw4ET4+PpgxY4bUZ/369RBC4N133y3zfDMzM+zZswd+fn7w9PTEhAkT4Ovri927d8PExETqt27dOnh7e8PX1xe+vr547bXXsGbNGv2/QCIiIqoWVEIIYewiqoOcnByo1WpotVrY2NhU2M99yvYX3tfV+b1feBtERET07N/f1WIOEREREZE+MRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIZNRAdOHAAffv2hYuLC1QqFTZv3qyzfuTIkVCpVDpL27ZtdfoUFhYiODgYDg4OsLS0RL9+/XDz5k2dPtnZ2QgICIBarYZarUZAQADu3r2r51dHRERE1YVRA1FeXh5atGiBpUuXVtinV69eSE9Pl5YdO3borA8JCUF0dDTWr1+PQ4cOITc3F3369EFJSYnUZ9iwYUhKSkJMTAxiYmKQlJSEgIAAvb0uIiIiql5qGnPn/v7+8Pf3f2Ifc3NzaDSactdptVqsXLkSa9asQY8ePQAAa9euhaurK3bv3g0/Pz+cP38eMTExiI+PR5s2bQAAK1asQLt27ZCcnAxPT8+qfVFERERU7ch+DlFsbCwcHR3RuHFjBAUFITMzU1qXkJCA4uJi+Pr6Sm0uLi7w8vJCXFwcAODIkSNQq9VSGAKAtm3bQq1WS33KU1hYiJycHJ2FiIiIXk6yDkT+/v5Yt24d9u7di4ULF+LPP/9Et27dUFhYCADIyMiAmZkZbG1tdZ7n5OSEjIwMqY+jo2OZbTs6Okp9yhMWFibNOVKr1XB1da3CV0ZERERyYtRDZk8zZMgQ6c9eXl5o3bo13NzcsH37dgwcOLDC5wkhoFKppMeP/rmiPo+bOnUqJk6cKD3OyclhKCIiInpJyXqE6HHOzs5wc3NDSkoKAECj0aCoqAjZ2dk6/TIzM+Hk5CT1uXXrVpltZWVlSX3KY25uDhsbG52FiIiIXk7VKhDduXMHN27cgLOzMwCgVatWMDU1xa5du6Q+6enpOHPmDNq3bw8AaNeuHbRaLY4dOyb1OXr0KLRardSHiIiIlM2oh8xyc3Nx6dIl6XFqaiqSkpJgZ2cHOzs7hIaGYtCgQXB2dsbVq1fxxRdfwMHBAQMGDAAAqNVqBAYGYtKkSbC3t4ednR0mT54Mb29v6ayzpk2bolevXggKCsLy5csBAGPGjEGfPn14hhkREREBMHIgOn78OLp27So9fjhnZ8SIEfjuu+9w+vRpREVF4e7du3B2dkbXrl2xYcMGWFtbS88JDw9HzZo1MXjwYBQUFKB79+6IiIiAiYmJ1GfdunWYMGGCdDZav379nnjtIyIiIlIWlRBCGLuI6iAnJwdqtRparfaJ84ncp2x/4X1dnd/7hbdBREREz/79Xa3mEBERERHpAwMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKV6VBKKSkhIkJSUhOzu7KjZHREREZFCVCkQhISFYuXIlgAdhqHPnzmjZsiVcXV0RGxtblfURERER6V2lAtHGjRvRokULAMBvv/2G1NRUXLhwASEhIZg2bVqVFkhERESkb5UKRLdv34ZGowEA7NixA++88w4aN26MwMBAnD59+pm3c+DAAfTt2xcuLi5QqVTYvHmztK64uBiff/45vL29YWlpCRcXF7z33nv466+/dLbRpUsXqFQqnWXo0KE6fbKzsxEQEAC1Wg21Wo2AgADcvXu3Mi+diIiIXkKVCkROTk44d+4cSkpKEBMTgx49egAA8vPzYWJi8szbycvLQ4sWLbB06dIy6/Lz85GYmIjp06cjMTERmzZtwsWLF9GvX78yfYOCgpCeni4ty5cv11k/bNgwJCUlISYmBjExMUhKSkJAQMBzvmoiIiJ6WdWszJNGjRqFwYMHw9nZGSqVCj179gQAHD16FE2aNHnm7fj7+8Pf37/cdWq1Grt27dJpW7JkCd58801cv34d9erVk9otLCykEavHnT9/HjExMYiPj0ebNm0AACtWrEC7du2QnJwMT0/PZ66XiIiIXk6VGiEKDQ3Fjz/+iDFjxuDw4cMwNzcHAJiYmGDKlClVWuCjtFotVCoV6tSpo9O+bt06ODg4oHnz5pg8eTLu3bsnrTty5AjUarUUhgCgbdu2UKvViIuLq3BfhYWFyMnJ0VmIiIjo5VSpESIA+Ne//gUA+Oeff6S2ESNGvHhFFfjnn38wZcoUDBs2DDY2NlL78OHD4eHhAY1GgzNnzmDq1Kk4efKkNLqUkZEBR0fHMttzdHRERkZGhfsLCwvDrFmzqv6FEBERkexUaoSopKQEc+bMwSuvvAIrKytcuXIFADB9+nTpdPyqVFxcjKFDh6K0tBTLli3TWRcUFIQePXrAy8sLQ4cOxcaNG7F7924kJiZKfVQqVZltCiHKbX9o6tSp0Gq10nLjxo2qe0FEREQkK5UKRF999RUiIiKwYMECmJmZSe3e3t748ccfq6w44EEYGjx4MFJTU7Fr1y6d0aHytGzZEqampkhJSQEAaDQa3Lp1q0y/rKwsODk5Vbgdc3Nz2NjY6CxERET0cqpUIIqKisIPP/yA4cOH65xV9tprr+HChQtVVtzDMJSSkoLdu3fD3t7+qc85e/YsiouL4ezsDABo164dtFotjh07JvU5evQotFot2rdvX2W1EhERUfVVqTlEaWlpaNiwYZn20tJSFBcXP/N2cnNzcenSJelxamoqkpKSYGdnBxcXF/zrX/9CYmIitm3bhpKSEmnOj52dHczMzHD58mWsW7cOb731FhwcHHDu3DlMmjQJPj4+6NChAwCgadOm6NWrF4KCgqTT8ceMGYM+ffrwDDMiIiICUMkRoubNm+PgwYNl2n/55Rf4+Pg883aOHz8OHx8f6TkTJ06Ej48PZsyYgZs3b2Lr1q24efMmXn/9dTg7O0vLw7PDzMzMsGfPHvj5+cHT0xMTJkyAr68vdu/erTNytW7dOnh7e8PX1xe+vr547bXXsGbNmsq8dCIiInoJVWqEaObMmQgICEBaWhpKS0uxadMmJCcnIyoqCtu2bXvm7XTp0gVCiArXP2kdALi6umL//v1P3Y+dnR3Wrl37zHURERGRslRqhKhv377YsGEDduzYAZVKhRkzZuD8+fP47bffpIs0EhEREVUXlb4OkZ+fH/z8/KqyFiIiIiKjqNQI0Z9//omjR4+WaT969CiOHz/+wkURERERGVKlAtG4cePKvVBhWloaxo0b98JFERERERlSpQLRuXPn0LJlyzLtPj4+OHfu3AsXRURERGRIlQpE5ubm5V79OT09HTVrVnpaEhEREZFRVCoQ9ezZU7rX10N3797FF198wbPMiIiIqNqp1HDOwoUL0alTJ7i5uUkXVUxKSoKTkxMveEhERETVTqUC0SuvvIJTp05h3bp1OHnyJGrXro1Ro0bh3XffhampaVXXSERERKRXlZ7wY2lpiTFjxlRlLURERERGUelAdPHiRcTGxiIzMxOlpaU662bMmPHChREREREZSqUC0YoVK/Dhhx/CwcEBGo0GKpVKWvfwVh5ERERE1UWlAtHcuXPx1Vdf4fPPP6/qeoiIiIgMrlKn3WdnZ+Odd96p6lqIiIiIjKJSgeidd97Bzp07q7oWIiIiIqOo1CGzhg0bYvr06YiPj4e3t3eZU+0nTJhQJcURERERGYJKCCGe90keHh4Vb1ClwpUrV16oKDnKycmBWq2GVquFjY1Nhf3cp2x/4X1dnd/7hbdBREREz/79XakRotTU1EoXRkRERCQ3lZpD9FBRURGSk5Nx//79qqqHiIiIyOAqFYjy8/MRGBgICwsLNG/eHNevXwfwYO7Q/Pnzq7RAIiIiIn2rVCCaOnUqTp48idjYWNSqVUtq79GjBzZs2FBlxREREREZQqXmEG3evBkbNmxA27Ztda5S3axZM1y+fLnKiiMiIiIyhEqNEGVlZcHR0bFMe15enk5AIiIiIqoOKhWI3njjDWzf/n+nlz8MQStWrEC7du2qpjIiIiIiA6nUIbOwsDD06tUL586dw/379/H111/j7NmzOHLkCPbv31/VNRIRERHpVaVGiNq3b4+4uDjk5+ejQYMG2LlzJ5ycnHDkyBG0atWqqmskIiIi0qvnHiEqLi7GmDFjMH36dERGRuqjJiIiIiKDeu4RIlNTU0RHR+ujFiIiIiKjqNQhswEDBmDz5s1VXAoRERGRcVT6bvdz5sxBXFwcWrVqBUtLS531vNs9ERERVSeVCkQ//vgj6tSpg4SEBCQkJOisU6lUDERERERUrfBu90RERKR4L3S3+xd14MAB9O3bFy4uLlCpVGXmJQkhEBoaChcXF9SuXRtdunTB2bNndfoUFhYiODgYDg4OsLS0RL9+/XDz5k2dPtnZ2QgICIBarYZarUZAQADu3r2r51dHRERE1UWlRohGjx79xPWrVq16pu3k5eWhRYsWGDVqFAYNGlRm/YIFC7Bo0SJERESgcePGmDt3Lnr27Ink5GRYW1sDAEJCQvDbb79h/fr1sLe3x6RJk9CnTx8kJCTAxMQEADBs2DDcvHkTMTExAIAxY8YgICAAv/322/O8bCIiInpJVSoQZWdn6zwuLi7GmTNncPfuXXTr1u2Zt+Pv7w9/f/9y1wkhsHjxYkybNg0DBw4EAERGRsLJyQk//fQTPvjgA2i1WqxcuRJr1qxBjx49AABr166Fq6srdu/eDT8/P5w/fx4xMTGIj49HmzZtAPzfLUaSk5Ph6elZmbeAiIiIXiKVCkTlXYeotLQUH330EerXr//CRQEP5illZGTA19dXajM3N0fnzp0RFxeHDz74AAkJCSguLtbp4+LiAi8vL8TFxcHPzw9HjhyBWq2WwhAAtG3bFmq1GnFxcRUGosLCQhQWFkqPc3JyquR1ERERkfxU2RyiGjVq4JNPPkF4eHiVbC8jIwMA4OTkpNPu5OQkrcvIyICZmRlsbW2f2MfR0bHM9h0dHaU+5QkLC5PmHKnVari6ur7Q6yEiIiL5qtJJ1ZcvX8b9+/ercpNQqVQ6j4UQZdoe93if8vo/bTtTp06FVquVlhs3bjxn5URERFRdVOqQ2cSJE3UeCyGQnp6O7du3Y8SIEVVSmEajAfBghMfZ2Vlqz8zMlEaNNBoNioqKkJ2drTNKlJmZifbt20t9bt26VWb7WVlZZUafHmVubg5zc/MqeS1EREQkb5UaITpx4oTOcurUKQDAwoULsXjx4iopzMPDAxqNBrt27ZLaioqKsH//finstGrVCqampjp90tPTcebMGalPu3btoNVqcezYManP0aNHodVqpT5ERESkbJUaIdq3b1+V7Dw3NxeXLl2SHqempiIpKQl2dnaoV68eQkJCMG/ePDRq1AiNGjXCvHnzYGFhgWHDhgEA1Go1AgMDMWnSJNjb28POzg6TJ0+Gt7e3dNZZ06ZN0atXLwQFBWH58uUAHpx236dPH55hRkRERABe4ErV9+/fR6NGjXTaU1JSYGpqCnd392fazvHjx9G1a1fp8cNDcSNGjEBERAQ+++wzFBQU4KOPPkJ2djbatGmDnTt3StcgAoDw8HDUrFkTgwcPRkFBAbp3746IiAjpGkQAsG7dOkyYMEE6G61fv35YunRpZV46ERERvYRUQgjxvE/q3LkzRo8eXWa+0Nq1a/Hjjz8iNja2quqTjZycHKjVami1WtjY2FTYz33K9hfe19X5vV94G0RERPTs39+VnkPUoUOHMu1t27ZFUlJSZTZJREREZDSVCkQqlQr37t0r067ValFSUvLCRREREREZUqUC0f/7f/8PYWFhOuGnpKQEYWFh6NixY5UVR0RERGQIlZpUvWDBAnTq1Amenp74f//v/wEADh48iJycHOzdu7dKCyQiIiLSt0qNEDVr1gynTp3C4MGDkZmZiXv37uG9997DhQsX4OXlVdU1EhEREelVpUaIgAc3UZ03b15V1kJERERkFJUaIVq9ejV++eWXMu2//PILIiMjX7goIiIiIkOqVCCaP38+HBwcyrQ7Ojpy1IiIiIiqnUoFomvXrsHDw6NMu5ubG65fv/7CRREREREZUqUCkaOjo3RD10edPHkS9vb2L1wUERERkSFVKhANHToUEyZMwL59+1BSUoKSkhLs3bsXH3/8MYYOHVrVNRIRERHpVaXOMps7dy6uXbuG7t27o2bNB5soKSnBiBEjOIeIiIiIqp1KBSIzMzNs2LABkydPRmpqKiwsLODt7Q03N7eqro+IiIhI7547EN29exfTpk3Dhg0bkJ2dDQCwtbXF0KFDMXfuXNSpU6eqayQiIiLSq+cKRH///TfatWuHtLQ0DB8+HE2bNoUQAufPn0dERAT27NmDuLg42Nra6qteIiIioir3XIFo9uzZMDMzw+XLl+Hk5FRmna+vL2bPno3w8PAqLZKIiIhIn57rLLPNmzfjP//5T5kwBAAajQYLFixAdHR0lRVHREREZAjPFYjS09PRvHnzCtd7eXkhIyPjhYsiIiIiMqTnCkQODg64evVqhetTU1N5YUYiIiKqdp4rEPXq1QvTpk1DUVFRmXWFhYWYPn06evXqVWXFERERERnCc02qnjVrFlq3bo1GjRph3LhxaNKkCQDg3LlzWLZsGQoLC7FmzRq9FErPz33K9irZztX5vatkO0RERHL1XIHo1VdfxZEjR/DRRx9h6tSpEEIAAFQqFXr27ImlS5fC1dVVL4USERER6ctzX5jRw8MDv//+O7Kzs5GSkgIAaNiwIezs7Kq8OCIiIiJDqNStO4AHV6d+8803q7IWIiIiIqOo1N3uiYiIiF4mDERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4lb4OEdHz4G1EiIhIzmQ/QuTu7g6VSlVmGTduHABg5MiRZda1bdtWZxuFhYUIDg6Gg4MDLC0t0a9fP9y8edMYL4eIiIhkSPaB6M8//0R6erq07Nq1CwDwzjvvSH169eql02fHjh062wgJCUF0dDTWr1+PQ4cOITc3F3369EFJSYlBXwsRERHJk+wPmdWtW1fn8fz589GgQQN07txZajM3N4dGoyn3+VqtFitXrsSaNWvQo0cPAMDatWvh6uqK3bt3w8/PT3/FExERUbUg+xGiRxUVFWHt2rUYPXo0VCqV1B4bGwtHR0c0btwYQUFByMzMlNYlJCSguLgYvr6+UpuLiwu8vLwQFxdn0PqJiIhInmQ/QvSozZs34+7duxg5cqTU5u/vj3feeQdubm5ITU3F9OnT0a1bNyQkJMDc3BwZGRkwMzODra2tzracnJyQkZFR4b4KCwtRWFgoPc7Jyany10NERETyUK0C0cqVK+Hv7w8XFxepbciQIdKfvby80Lp1a7i5uWH79u0YOHBghdsSQuiMMj0uLCwMs2bNqprCiYiISNaqzSGza9euYffu3Xj//fef2M/Z2Rlubm5ISUkBAGg0GhQVFSE7O1unX2ZmJpycnCrcztSpU6HVaqXlxo0bL/4iiIiISJaqTSBavXo1HB0d0bv3k69Dc+fOHdy4cQPOzs4AgFatWsHU1FQ6Ow0A0tPTcebMGbRv377C7Zibm8PGxkZnISIiopdTtThkVlpaitWrV2PEiBGoWfP/Ss7NzUVoaCgGDRoEZ2dnXL16FV988QUcHBwwYMAAAIBarUZgYCAmTZoEe3t72NnZYfLkyfD29pbOOiMiIiJlqxaBaPfu3bh+/TpGjx6t025iYoLTp08jKioKd+/ehbOzM7p27YoNGzbA2tpa6hceHo6aNWti8ODBKCgoQPfu3REREQETExNDvxQiIiKSoWoRiHx9fSGEKNNeu3Zt/PHHH099fq1atbBkyRIsWbJEH+URERFRNVdt5hARERER6QsDERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpnqwDUWhoKFQqlc6i0Wik9UIIhIaGwsXFBbVr10aXLl1w9uxZnW0UFhYiODgYDg4OsLS0RL9+/XDz5k1DvxQiIiKSMVkHIgBo3rw50tPTpeX06dPSugULFmDRokVYunQp/vzzT2g0GvTs2RP37t2T+oSEhCA6Ohrr16/HoUOHkJubiz59+qCkpMQYL4eIiIhkqKaxC3iamjVr6owKPSSEwOLFizFt2jQMHDgQABAZGQknJyf89NNP+OCDD6DVarFy5UqsWbMGPXr0AACsXbsWrq6u2L17N/z8/Az6WoiIiEieZD9ClJKSAhcXF3h4eGDo0KG4cuUKACA1NRUZGRnw9fWV+pqbm6Nz586Ii4sDACQkJKC4uFinj4uLC7y8vKQ+FSksLEROTo7OQkRERC8nWQeiNm3aICoqCn/88QdWrFiBjIwMtG/fHnfu3EFGRgYAwMnJSec5Tk5O0rqMjAyYmZnB1ta2wj4VCQsLg1qtlhZXV9cqfGVEREQkJ7IORP7+/hg0aBC8vb3Ro0cPbN++HcCDQ2MPqVQqnecIIcq0Pe5Z+kydOhVarVZabty4UclXQURERHIn60D0OEtLS3h7eyMlJUWaV/T4SE9mZqY0aqTRaFBUVITs7OwK+1TE3NwcNjY2OgsRERG9nKpVICosLMT58+fh7OwMDw8PaDQa7Nq1S1pfVFSE/fv3o3379gCAVq1awdTUVKdPeno6zpw5I/UhIiIikvVZZpMnT0bfvn1Rr149ZGZmYu7cucjJycGIESOgUqkQEhKCefPmoVGjRmjUqBHmzZsHCwsLDBs2DACgVqsRGBiISZMmwd7eHnZ2dpg8ebJ0CI6IiIgIkHkgunnzJt59913cvn0bdevWRdu2bREfHw83NzcAwGeffYaCggJ89NFHyM7ORps2bbBz505YW1tL2wgPD0fNmjUxePBgFBQUoHv37oiIiICJiYmxXhYRERHJjKwD0fr165+4XqVSITQ0FKGhoRX2qVWrFpYsWYIlS5ZUcXVERET0sqhWc4iIiIiI9IGBiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUr6axCyAyBvcp26tkO1fn966S7RARkXFxhIiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI/3MiMyMt5XjYjI+DhCRERERIon60AUFhaGN954A9bW1nB0dMTbb7+N5ORknT4jR46ESqXSWdq2bavTp7CwEMHBwXBwcIClpSX69euHmzdvGvKlEBERkYzJOhDt378f48aNQ3x8PHbt2oX79+/D19cXeXl5Ov169eqF9PR0admxY4fO+pCQEERHR2P9+vU4dOgQcnNz0adPH5SUlBjy5RAREZFMyXoOUUxMjM7j1atXw9HREQkJCejUqZPUbm5uDo1GU+42tFotVq5ciTVr1qBHjx4AgLVr18LV1RW7d++Gn5+f/l4AERERVQuyHiF6nFarBQDY2dnptMfGxsLR0RGNGzdGUFAQMjMzpXUJCQkoLi6Gr6+v1Obi4gIvLy/ExcVVuK/CwkLk5OToLERERPRyqjaBSAiBiRMnomPHjvDy8pLa/f39sW7dOuzduxcLFy7En3/+iW7duqGwsBAAkJGRATMzM9ja2upsz8nJCRkZGRXuLywsDGq1WlpcXV3188KIiIjI6GR9yOxR48ePx6lTp3Do0CGd9iFDhkh/9vLyQuvWreHm5obt27dj4MCBFW5PCAGVSlXh+qlTp2LixInS45ycHIYiIiKil1S1GCEKDg7G1q1bsW/fPrz66qtP7Ovs7Aw3NzekpKQAADQaDYqKipCdna3TLzMzE05OThVux9zcHDY2NjoLERERvZxkHYiEEBg/fjw2bdqEvXv3wsPD46nPuXPnDm7cuAFnZ2cAQKtWrWBqaopdu3ZJfdLT03HmzBm0b99eb7UTERFR9SHrQ2bjxo3DTz/9hC1btsDa2lqa86NWq1G7dm3k5uYiNDQUgwYNgrOzM65evYovvvgCDg4OGDBggNQ3MDAQkyZNgr29Pezs7DB58mR4e3tLZ50RERGRssk6EH333XcAgC5duui0r169GiNHjoSJiQlOnz6NqKgo3L17F87OzujatSs2bNgAa2trqX94eDhq1qyJwYMHo6CgAN27d0dERARMTEwM+XKIiIhIpmQdiIQQT1xfu3Zt/PHHH0/dTq1atbBkyRIsWbKkqkojIiKil4isAxERGV5V3GyWN5oloupG1pOqiYiIiAyBgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj2eZEZFsVcUZbwDPeiOip+MIERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR4DERERESkeAxEREREpHgMRERERKR5v7kpE9Iyq4mazvNEskTxxhIiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPgYiIiIgUj4GIiIiIFI+BiIiIiBSPV6omIqqGquKq2QCvnE30EAMRERG9EIYzehnwkBkREREpnqIC0bJly+Dh4YFatWqhVatWOHjwoLFLIiIiIhlQzCGzDRs2ICQkBMuWLUOHDh2wfPly+Pv749y5c6hXr56xyyMioirCQ3hUGYoZIVq0aBECAwPx/vvvo2nTpli8eDFcXV3x3XffGbs0IiIiMjJFjBAVFRUhISEBU6ZM0Wn39fVFXFxcuc8pLCxEYWGh9Fir1QIAcnJynriv0sL8F6z26ft4VlVRC1A19cipFkBe9cipFoCf4Sfhe1M+OdUCyK8eMq6Hf49CiCd3FAqQlpYmAIjDhw/rtH/11VeicePG5T5n5syZAgAXLly4cOHC5SVYbty48cSsoIgRoodUKpXOYyFEmbaHpk6diokTJ0qPS0tL8ffff8Pe3r7C5zxNTk4OXF1dcePGDdjY2FRqG1VJTvXIqRa51SOnWuRWD2upHvXIqRa51SOnWuRWT1XVIoTAvXv34OLi8sR+ighEDg4OMDExQUZGhk57ZmYmnJycyn2Oubk5zM3Nddrq1KlTJfXY2NgY/YP2KDnVI6daAHnVI6daAHnVw1oqJqd65FQLIK965FQLIK96qqIWtVr91D6KmFRtZmaGVq1aYdeuXTrtu3btQvv27Y1UFREREcmFIkaIAGDixIkICAhA69at0a5dO/zwww+4fv06xo4da+zSiIiIyMgUE4iGDBmCO3fuYPbs2UhPT4eXlxd27NgBNzc3g9Vgbm6OmTNnljkUZyxyqkdOtQDyqkdOtQDyqoe1VExO9cipFkBe9cipFkBe9Ri6FpUQTzsPjYiIiOjlpog5RERERERPwkBEREREisdARERERIrHQERERESKx0BEREREisdARAbVsmVLZGdnAwBmz56N/PyquQkjERE9v4KCAp3/h69du4bFixdj586dRqzKOHjavQGkpaXh8OHDyMzMRGlpqc66CRMmGKmqB3JycrB37154enqiadOmet9f7dq1kZKSgldffRUmJiZIT0+Ho6Oj3vdbXeXn5+P69esoKirSaX/ttdeMVBFw48YNqFQqvPrqq0arobS0FJcuXSr331SnTp30vv+JEydizpw5sLS0xIEDB9C+fXvUrCmfy7plZmYiOTkZKpUKjRs35r+xRxQVFSE1NRUNGjSQ1d+Zsfj6+mLgwIEYO3Ys7t69iyZNmsDU1BS3b9/GokWL8OGHHxq0njt37mDGjBnYt29fuf++//77b73tm4FIz1avXo2xY8fCzMyszI1hVSoVrly5YtB6Bg8ejE6dOmH8+PEoKChAixYtcPXqVQghsH79egwaNEiv+2/Xrh2srKzQsWNHzJo1C5MnT4aVlVW5fWfMmKHXWh6XmJgIU1NTeHt7AwC2bNmC1atXo1mzZggNDYWZmZnBasnKysKoUaPw+++/l7u+pKTEYLUAwP379zFr1ix88803yM3NBQBYWVkhODgYM2fOhKmpqcFqiY+Px7Bhw3Dt2jU8/t+XSqUyyHtjamqKmzdvwsnJSVbBPicnB+PGjcP69eul98HExARDhgzBt99++0z3c6pKQghs3Lixwi+3TZs2GayW/Px8BAcHIzIyEgBw8eJF1K9fHxMmTICLiwumTJlisFry8vIwf/587Nmzp9z3xZDfCw4ODti/fz+aN2+OH3/8EUuWLMGJEyfw66+/YsaMGTh//rzBagEAf39/XL58GYGBgXBycipzM/URI0bobd+Mx3o2Y8YMzJgxA1OnTkWNGsY/QnngwAFMmzYNABAdHQ0hBO7evYvIyEjMnTtX74EoIiICM2fOxLZt26BSqfD777+X+ytNpVIZPBB98MEHmDJlCry9vXHlyhUMHToUAwYMwC+//IL8/HwsXrzYYLWEhIQgOzsb8fHx6Nq1K6Kjo3Hr1i3MnTsXCxcuNFgdD40fPx7R0dFYsGAB2rVrBwA4cuQIQkNDcfv2bXz//fcGq2Xs2LFo3bo1tm/fDmdn5zL/YRqCu7s7vvnmG/j6+kIIgSNHjsDW1rbcvoYYsXro/fffR1JSErZt24Z27dpBpVIhLi4OH3/8MYKCgvDzzz8brBYA+Pjjj/HDDz+ga9eu5X65GdLUqVNx8uRJxMbGolevXlJ7jx49MHPmTIMGovfffx/79+9HQECA0T7DD+Xn58Pa2hoAsHPnTgwcOBA1atRA27Ztce3aNYPXc+jQIRw6dAgtWrQw+L4hSK/s7OzEpUuXjF2GpFatWuL69etCCCECAgLE559/LoQQ4tq1a8LS0tKgtahUKnHr1i2D7vNJbGxspL+r+fPnC19fXyGEEIcOHRKvvvqqQWvRaDTi6NGjQgghrK2tRXJyshBCiC1btogOHToYtBYhHrw3O3bsKNO+Y8cOYWNjY9BaLCwsREpKikH3+bjo6Gjh5OQkVCqVqFGjhlCpVOUuNWrUMGhdFhYW4uDBg2XaDxw4ICwsLAxaixBC2Nraiu3btxt8v+WpV6+eOHLkiBBCCCsrK3H58mUhhBApKSnC2traoLWo1Wpx6NAhg+6zIt7e3uLrr78W169fFzY2NiIuLk4IIcTx48eFk5OTwetp3bq19PdkaMYfsnjJBQYG4pdffjF2GRJXV1ccOXIEeXl5iImJga+vLwAgOzsbtWrVMmgtpaWlsjjM8JAQQhq63r17N9566y0AD96z27dvG7SWvLw86b2xs7NDVlYWAMDb2xuJiYkGrQUAatWqBXd39zLt7u7uBj2UCABt2rTBpUuXDLrPx7399tvIyMhATk4OhBBITk5GdnZ2mUWf8x3KY29vX+5hMbVaXeEIlj6p1WrUr1/f4PstT1ZWVrn/3+Tl5Rl8hMbW1hZ2dnYG3WdFZsyYgcmTJ8Pd3R1vvvmmNAK8c+dO+Pj4GLyeZcuWYdq0adi/fz/u3LmDnJwcnUWfeMhMz8LCwtCnTx/ExMTA29u7zFyLRYsWGbSekJAQDB8+HFZWVnBzc0OXLl0APDiU9nDujCHJacJ569atMXfuXPTo0QP79+/Hd999BwBITU2Fk5OTQWvx9PREcnIy3N3d8frrr2P58uVwd3fH999/D2dnZ4PWAgDjxo3DnDlzsHr1aulGi4WFhfjqq68wfvx4g9YSHByMSZMmISMjo9x/U4accG5lZYV9+/bBw8NDFhN0v/zyS0ycOBFRUVHS5yQjIwOffvoppk+fbvB6QkNDMWvWLKxatQq1a9c2+P4f9cYbb2D79u0IDg4GACkErVixQgoBhjJnzhzMmDEDkZGRsLCwMOi+H/evf/0LHTt2RHp6us5hqu7du2PAgAEGr6dOnTrQarXo1q2bTrsQQu9zBDmpWs/mzJmDmTNnwtPTs8wxdJVKhb179xq8puPHj+PGjRvo2bOnNKF5+/btqFOnDjp06GCwOuQ24fzUqVMYPnw4rl+/jokTJ2LmzJkAHnwB37lzBz/99JPBalm3bh2Ki4sxcuRInDhxAn5+frhz5w7MzMwQERGBIUOGGKwWABgwYAD27NkDc3Nz6T/NkydPoqioCN27d9fpq++JsuXNxVOpVAb5D1PufHx8cOnSJRQWFqJevXoAgOvXr8Pc3ByNGjXS6WuIkcb8/HwMHDgQhw8fhru7e5nwasjRzri4OPTq1QvDhw9HREQEPvjgA5w9exZHjhzB/v370apVK4PV4uPjg8uXL0MIYfT35VFyOIP0zTffRM2aNfHxxx+XO++sc+fOets3A5Ge2draIjw8HCNHjjR2KWU8/Ks31oQ+V1dXjB07VjYTzivyzz//wMTExKBnUj0uPz8fFy5cQL169eDg4GDw/Y8aNeqZ+65evVqPleCpEz3d3Nz0uv/ydO3aFW5uboiIiJDaRowYgRs3bhj0R8+sWbOeue/DwK9PgwcPxr59+/Cvf/2r3C83Q9TwqNOnT+M///kPEhISUFpaipYtW+Lzzz83+Oj40/6eDPm+yOkMUgCwsLDAiRMn4OnpadD9Ajxkpnfm5uYGHXV5FitXrkR4eDhSUlIAAI0aNUJISAjef/99g9aRn5+PoUOHyjoMATD43KryWFhYoGXLlkbbv75DzvMwRuB5Gnd39zKHMl955RWDf7YNHTCeZvv27fjjjz/QsWNHY5cC4MEcvIen3RuTnP6e5HQGKfBg6sKNGzeMEog4QqRnYWFhSE9PxzfffGPsUgAA06dPR3h4OIKDg3U+/EuXLsXHH3+MuXPnGqyWzz77DHZ2dgY93fVpRo8eDWdnZ3z11VdS2xdffIGMjAysWrXKYHWUlJQgIiKiwuuUGPpQa0FBAYQQ0nyHa9euITo6Gs2aNZMm5htKVFTUE9e/9957BqqEnqZJkyb4+eefjXohUXoytVqN9evXw9/fX6f9999/x9ChQ6HVag1azy+//ILQ0FB8+umnBp8jyECkZwMGDMDevXthb2+P5s2bl/nLNeSFyYAHF+FasmQJ3n33XZ32//73vwgODjbo2VQlJSXo06cPCgoKZDHhHJDPoY/x48cjIiICvXv3Lvc6JeHh4QarBSh7NVtPT0+YmZkZ5Wq2j58tVVxcjPz8fJiZmcHCwsLgZ3bJjVxCPfBghGjJkiX4/vvvyz1L0dCM+e/bzs4OFy9ehIODA2xtbZ84VcGQn2EnJyfExsaWuVPB+fPn0alTJ+kMV0Mx5hxBHjLTszp16mDgwIHGLkNSUlKC1q1bl2lv1aoV7t+/b9Ba5s2bhz/++EMaGn18UrUx7Nu3r0ybMYbY169fj59//lk69d/YEhMTpRC2ceNGaDQanavZGjIQPbwX3qNSUlLw4Ycf4tNPPzVYHQ9VNPqrUqlQq1YtNGzYEJ06dYKJiYlB6klNTS0zopiWloYbN24YZP+P+p//+R/k5+ejQYMGsLCwKPOjx9Dh1ZiHNsPDw6ULIBryIq9PI6czSIEHn19j4QiRHmzduhX+/v5GnYRbkeDgYJiampYZfZk8eTIKCgrw7bffGqwWOU84NzYXFxfExsaicePGxi4FwIM5TA8ndQ8ePBjNmzfHzJkzpWP9crhJ7/Hjx/E///M/uHDhgkH36+HhgaysLOTn58PW1la6+ruFhQWsrKyQmZmJ+vXrY9++fXB1dTVobcb2tB8T+rwNAz0bOZxB2rJlS+zZswe2traYPXs2Jk+ebJTLETAQ6YGJiQkyMjJQt25dWd3nCHgQiKKiouDq6oq2bdsCeHBvqBs3buC9997TCXH6PmSl0Whw8ODBMqcDG9PFixcRGxtb7rwdQ95KZOHChbhy5QqWLl1q1Mv6P/Taa6/h/fffx4ABA+Dl5YWYmBi0a9cOCQkJ6N27NzIyMoxdIk6cOIHOnTvr/eJtj/vvf/+LH374AT/++CMaNGgAALh06RI++OADjBkzBh06dMDQoUOh0WiwceNGg9ZG9DRyOINULjf9ZiDSA41GgxUrVqBv376oUaMGbt26hbp16xq7LAAPjqE/C0NcI0luE85XrFiBDz/8EA4ODtBoNGUO4Rny2iADBgzAvn37YGdnJ4u5Zxs3bsSwYcNQUlKC7t27Y+fOnQAe/B0eOHCgwpvQ6sPWrVt1HgshkJ6ejqVLl8LV1dWgtQBAgwYN8Ouvv+L111/XaT9x4gQGDRqEK1euIC4uDoMGDUJ6erre65FLqJcbuR3alMt8RTmQy02/OYdID8aOHYv+/ftDpVJBpVJBo9FU2NfQF5Erb46MsRw7dgx79+7Ftm3bZPGlP3fuXHz11Vf4/PPPDbrf8tSpU8coV4mtiJyuZvv222/rPFapVKhbty66detmlBvfpqenlzv/7v79+9LImYuLC+7du6f3Wp4W6o0RiOTyxR8eHi6rQ5tyuVSDHMjlpt8cIdKTCxcu4NKlS+jXrx9Wr16NOnXqlNuvf//+hi1MRp42VGvoa9/Y2NggKSlJNvdeourh4SHDH3/8Ubr304kTJxAUFASNRoNt27bht99+wxdffIHTp0/rtRY3Nzd89NFHsgj1D40aNQrOzs6YN2+e1PbFF18gPT3doP/GeWizYrNnz4aDgwM++ugjqW3ZsmW4ffu2wUN0jRo1kJGRYZxpJga6iaxihYaGiry8PGOXoSMyMlJs3rxZp23z5s0iMjLSSBXJw+jRo8V3331n7DJkS46fm9LSUlFaWmq0/QshRHp6uujRo4dQqVTCzMxMmJmZCZVKJXr27CnS09OFEELs3btX/PHHH3qvxdraWrqLO+mqX7++OHHiRJn2xMRE4eHhIYQQ4vDhw0Kj0Ri4MuNzd3cXPXr00Gnr1q2b9L4oBUeIFKhGjRpo0qQJzp07J7U1adIEKSkpir4PVFhYGBYtWoTevXuXe10kQ99sVk6/2gB5fW6ioqLw73//W7raeuPGjfHpp58iICDAoHU8Kjk5GcnJyRBCoEmTJka50m5gYCDeeOMNjB071uD7ljsLCwscOHCgzGVH/vzzT3Tu3Bn5+fm4evUqvLy8pFtY6Ivc5jPRAwxEBiC3LzY5kdN74+HhUeE6Y9xs1sPDAw0bNsSuXbuktu7duyM1NdXgtcjJokWLMH36dIwfPx4dOnSAEAKHDx/Gt99+i7lz5+KTTz7Rew0TJ07EnDlzYGlpiYkTJz61XkORW6iX0xe/nA5t8lINT1ajRg00bdoUZ8+eldqaNm2Kixcv8m731R2/2CrG94ael4eHB2bNmlXmFh2RkZEIDQ01yIXdunbtiujoaNSpU+eJZ24a4mzNR8kx1Mvliz8jIwMBAQHYs2ePFBSLi4vRo0cPREVFQaPRYN++fSguLtb77WjkOJ8pPz8f169fR1FRkU67MW67EhERgTp16uicQLF582ZotVq9XruKgUjBCgoKUFxcrNNmY2NjpGqM41l/6atUKqOcwSRHxv7c1KpVC2fOnEHDhg112lNSUuDt7Y1//vnHYLXQk8nxi18OhzbldKmGrKwsjBo1qsLLVShpGgVPu1eY/Px8fPbZZ/j5559x586dMuuV9OEHHvwH9PDL/cSJExX2M9bFEeXyq01On5uGDRvi559/xhdffKHTvmHDBlld5NNQ5Bzqv/zyS/z6669SGAIe/P395z//kb74FyxYgEGDBull/097b/bv3y/92ZCHNuV0qYaQkBBkZ2cjPj5eGvm8desW5s6da5QfgaNHj8bXX38t3ebkoby8PAQHB+v1fnwMRAYily+2Tz/9FPv27cOyZcvw3nvv4dtvv0VaWhqWL1+O+fPnG7SWh4z53jx6XSY5XaNJbr/a5PS5mTVrFoYMGYIDBw6gQ4cOUKlUOHToEPbs2YOff/7ZoLXIgZxDvbG/+OX63nTt2hUffPBBmflMH374Ibp16wYAOH369BMPgVaVvXv3YsuWLXjjjTdQo0YNuLm5oWfPnrCxsUFYWBh69+6t9xoeFRkZifnz55cJRAUFBYiKitLvDYqNcm6bgmRmZorevXuLGjVqlLsYmqurq9i3b58Q4sEpuikpKUIIIaKiooS/v79Ba5HbeyMnw4YNE+3btxfHjh0TlpaWYufOnWLNmjXC09NTbNu2zeD1yOlzI4QQx48fF8OHDxctW7YUPj4+Yvjw4SIxMdHgddCTvfXWW6Jly5Y6fzeJiYmiVatWonfv3kIIIbZu3Sq8vLyMVaJRyO1SDampqUIIIdzc3MShQ4eEEEJcuXJF1K5dW+/7f0ir1Yq7d+8KlUolLl26JLRarbT8/fffIjIyUjg7O+u1BgYiPZPbF5ulpaW4evWqEEKIV155RRw9elQI8eDDb2lpadBa5PbeyIlGo5H+bqytrUVycrIQQogtW7aIDh06GLweOX1uqPqQ0xe/HF24cEFs2bJFbN68WVy4cMEoNbRu3VrExMQIIYTo37+/CAgIEDdv3hSfffaZqF+/vsHqUKlUFf44rlGjhjAxMRFz587Vaw08ZKZnchuOrF+/Pq5evQo3Nzc0a9YMP//8M95880389ttvFV5NW1/k9t7ISV5ennSlVjs7O2RlZaFx48bw9vY26D3VHpLT5wYASktLcenSpXLv19WpUyeD10Pl02g02LVr1xMnMj/r/RWrO7nOZwoJCZEmbs+cORN+fn5Yu3YtzMzMEBkZabA69u3bByEEunXrhl9//RV2dnbSOjMzM7i5ucHFxUWvNTAQ6ZncvthGjRqFkydPonPnzpg6dSp69+6NJUuW4P79+wb9RwjI772RE09PTyQnJ8Pd3R2vv/46li9fDnd3d3z//fdl7n9kCHL63MTHx2PYsGG4du0axGMnyapUKsWdGCA3cv3iNza5zmcaPny49GcfHx9cvXoVFy5cQL169eDg4GCwOjp37gwASE1Nhaurq1Hu6cZApGdy+2J79KJ1Xbt2xYULF3D8+HE0aNBA56adhiC390ZO5PKr7SE5fW7Gjh2L1q1bY/v27XB2djbaGYBUPrl+8RubXE/gqOjMxEcvntm/f3+dERt9cnNzA2Cck214HSI9W7duHYqLizFy5EicOHECfn5+uH37tvTFNmTIEIPXtGfPHuzZs6fcww16ncH/GDm+N3KVn59vlF9tj5LL58bS0hInT54scx0iInp+Xbt2RWJiIkpKSuDp6QkhBFJSUmBiYoImTZogOTlZOpOzWbNmeq/HmGfYcoRIz+QyHPnQrFmzMHv2bLRu3drov67l9t7Iidx+tcnpc9OmTRtcunSJgYioCjz8f2T16tXSBVZzcnIQGBiIjh07IigoCMOGDcMnn3yCP/74Q+/1GPO6SBwh0jO5fbE5OztjwYIFRr0J5kNye2/kRG6/2uT0uYmOjsaXX36JTz/9tNz7dRnjVgNE1dUrr7yCXbt2lfl/5OzZs/D19UVaWhoSExPh6+uL27dv670eZ2dnbNmyBW+++SZsbGxw/PhxNG7cGFu3bsWCBQtw6NAhve2bgUjP5PbFZm9vj2PHjulcOdZY5PbeyMnixYtx8ODBp/5qKygoMMivNjl9bsqbbKlSqSCE4KRqoudkZWWFbdu2oUuXLjrtsbGx6Nu3L+7du4crV67g9ddfR05Ojt7rsbGxwalTp+Du7g53d3esW7cOHTp0QGpqKpo3b478/Hy97dvw07gVpn///ujRowf++usvJCQkIDExEWlpaejZsyfeffddpKWloVOnTga5QzcAvP/++/jpp58Msq+nkdt7Iyf//ve/MWfOHJ17hNnY2CA0NBQLFiyAhYUFZsyYgYSEBIPUI6fPTWpqapnlypUrvCEwUSX0798fo0ePRnR0NG7evIm0tDRER0cjMDBQurnqsWPH0LhxY4PU8/BkGwDSyTZpaWkGOdmGI0R6JrfhyI8//hhRUVF47bXX8Nprr5U53GDI02Dl9t7Iidx+tcnpc0NEVSc3NxeffPIJoqKipNus1KxZEyNGjEB4eDgsLS2RlJQEAGVuRqsPxjzZhpOq9Uyr1SIzM7PMl35WVpb0RVanTp0ypxbqy6lTp6QP9ZkzZ3TWGXqirNzeGzl5+Ktt4cKFeOONN6BSqXDs2DFMnjzZKL/a5PS5IaKqY2VlhRUrViA8PBxXrlyBEAINGjSAlZWV1McQQeghY55sw0CkZ3L7YpPT9S/k9t7IyfLly/HJJ59g6NCh5f5qA4AmTZrgxx9/NEg9cvrcAA/mETVt2hRnz56V2po2bYqLFy9yDhFRJVhZWcnihARjnmzDQ2Z6JrfhSDnhe/N0ubm5Ff5qU7KIiAjUqVNHCs4AsHnzZmi1WowYMcJ4hRHRCzHmyTYMRAbCL7aK8b0hIiLAuGfYMhARUbUyevRofP3117C2ttZpz8vLQ3BwsEGvmk1EVcuYJ9vwtHsiqlYiIyNRUFBQpr2goABRUVFGqIiIqsrDk20eZ4iTbTipmoiqhZycHAghIITAvXv3UKtWLWldSUkJduzYAUdHRyNWSEQvypgn2/CQGRFVCzVq1HjiKf4qlQqzZs3CtGnTDFgVEVUlY55sw0BERNXC/v37IYRAt27d8Ouvv+qcdmtmZgY3Nze4uLgYsUIiqirGONmGgYiIqpVr167B1dW13HuaERFVFgMREVVL+fn5uH79epnJlXK4uBwRVT+cVE1E1UpWVhZGjRqF33//vdz1vFI1EVUGx5yJqFoJCQlBdnY24uPjUbt2bcTExCAyMhKNGjXC1q1bjV0eEVVTHCEiompl79692LJlC9544w3UqFEDbm5u6NmzJ2xsbBAWFobevXsbu0QiqoY4QkRE1UpeXp50vSE7OztkZWUBALy9vZGYmGjM0oioGmMgIqJqxdPTE8nJyQAeXIdk+fLlSEtLw/fffw9nZ2cjV0dE1RXPMiOiamXdunUoLi7GyJEjceLECfj5+eH27dswMzNDZGQkhgwZYuwSiagaYiAiomotPz8fFy5cQL169eDg4GDscoiomuKkaiKqViZOnFhuu0qlQq1atdCwYUP0799f50rWRERPwxEiIqpWunbtisTERJSUlMDT0xNCCKSkpMDExARNmjRBcnIyVCoVDh06hGbNmhm7XCKqJjipmoiqlf79+6NHjx7466+/kJCQgMTERKSlpaFnz5549913kZaWhk6dOuGTTz4xdqlEVI1whIiIqpVXXnkFu3btKjP6c/bsWfj6+iItLQ2JiYnw9fXF7du3jVQlEVU3HCEiompFq9UiMzOzTHtWVhZycnIAAHXq1ClzjzMioidhICKiaqV///4YPXo0oqOjcfPmTaSlpSE6OhqBgYF4++23AQDHjh1D48aNjVsoEVUrPGRGRNVKbm4uPvnkE0RFReH+/fsAgJo1a2LEiBEIDw+HpaUlkpKSADy4cCMR0bNgICKiaik3NxdXrlyBEAINGjSAlZWVsUsiomqMgYiIiIgUj3OIiIiISPEYiIiIiEjxGIiIiIhI8RiIiKha6dSpE3766SfpcUZGBnr27AlLS0vUqVMHwIP7mm3evNk4Bb7Eli5din79+hm7DCK9YCAiekllZGTg448/RsOGDVGrVi04OTmhY8eO+P7775Gfn2/s8ipl27ZtyMjIwNChQ6W28PBwpKenIykpCRcvXgQApKenw9/fv9L7iY2NhUqlwt27d5/YLyIiQgphhjRy5EjpmkuGFBQUhD///BOHDh0y+L6J9I13uyd6CV25cgUdOnRAnTp1MG/ePHh7e+P+/fu4ePEiVq1aBRcXlyr9pV9cXAxTU1OdtqKiIpiZmT33tp70vG+++QajRo1CjRr/91vu8uXLaNWqFRo1aiS1aTSa566Xns7c3BzDhg3DkiVL0LFjR2OXQ1S1BBG9dPz8/MSrr74qcnNzy11fWlr6xOevWrVKNGnSRJibmwtPT0/x7bffSutSU1MFALFhwwbRuXNnYW5uLlatWiVGjBgh+vfvL+bNmyecnZ2Fm5ubEEKIU6dOia5du4patWoJOzs7ERQUJO7duydtr6LnPS4rK0uoVCpx5swZqc3NzU0AkJYRI0YIIYQAIKKjo59Y79WrV0WfPn1EnTp1hIWFhWjWrJnYvn271L+87T5q3759ZfrNnDlTfPPNN8LLy0vqFx0dLQCIpUuXSm2+vr5iypQp0uNly5aJ+vXrC1NTU9G4cWMRFRVV4d/NzJkzy+x33759QgghPvvsM9GoUSNRu3Zt4eHhIb788ktRVFSk8/w5c+aIunXrCisrKxEYGCg+//xz0aJFC53X9cYbbwgLCwuhVqtF+/btxdWrV6X1sbGxwszMTOTn51dYI1F1xEBE9JK5ffu2UKlUIiwsrFLP/+GHH4Szs7P49ddfxZUrV8Svv/4q7OzsREREhBDi/wKGu7u71CctLU2MGDFCWFlZiYCAAHHmzBlx+vRpkZeXJ1xcXMTAgQPF6dOnxZ49e4SHh4dOwCjveeWJjo4WlpaWoqSkRGrLzMwUvXr1EoMHDxbp6eni7t27QojyA9Hj9fbu3Vv07NlTnDp1Sly+fFn89ttvYv/+/eL+/fvi119/FQBEcnKyznYfVVhYKBYvXixsbGxEenq6SE9PF/fu3ROnTp0SKpVKZGVlCSGECAkJEQ4ODuKdd94RQghRXFwsrKysxO+//y6EEGLTpk3C1NRUfPvttyI5OVksXLhQmJiYiL1795b7Pty7d08MHjxY9OrVS9pvYWGhEOJB2Dl8+LBITU0VW7duFU5OTuJ///d/peeuXbtW1KpVS6xatUokJyeLWbNmCRsbGykQFRcXC7VaLSZPniwuXbokzp07JyIiIsS1a9ekbeTm5gqVSiViY2PL/wARVVMMREQvmfj4eAFAbNq0Safd3t5eWFpaCktLS/HZZ59V+HxXV1fx008/6bTNmTNHtGvXTgjxfwFj8eLFOn1GjBghnJycpC9nIR6EK1tbW52Rqu3bt4saNWqIjIyMCp9XnvDwcFG/fv0y7f379y8zglNeIHq8Xm9vbxEaGlruvh6O/mRnZz+xptWrVwu1Wq3TVlpaKhwcHMTGjRuFEEK8/vrrIiwsTDg6OgohhIiLixM1a9aURsnat28vgoKCdLbxzjvviLfeeqvC/T4cVXuaBQsWiFatWkmP27RpI8aNG6fTp0OHDlIgunPnjgDw1LBja2srBWSilwUnVRO9pFQqlc7jY8eOISkpCc2bN0dhYWG5z8nKysKNGzcQGBgIKysraZk7dy4uX76s07d169Zlnu/t7a0z/+f8+fNo0aIFLC0tpbYOHTqgtLQUycnJFT6vPAUFBahVq9YT+zzJ4/VOmDABc+fORYcOHTBz5kycOnWq0tt+lEqlQqdOnRAbG4u7d+/i7NmzGDt2LEpKSnD+/HnExsaiZcuW0q1Gzp8/jw4dOuhso0OHDjh//vxz73vjxo3o2LEjNBoNrKysMH36dFy/fl1an5ycjDfffFPnOY8+trOzw8iRI+Hn54e+ffvi66+/Rnp6epn91K5du9pOzCeqCAMR0UumYcOGUKlUuHDhgk57/fr10bBhQ9SuXbvC55aWlgIAVqxYgaSkJGk5c+YM4uPjdfo+GnIqahNClAlmDz3aXt62Hufg4IDs7Oyn9qvI4/t4//33ceXKFQQEBOD06dNo3bo1lixZUuntP6pLly6IjY3FwYMH0aJFC9SpUwedOnXC/v37ERsbiy5duuj0f/w9etL7VpH4+HgMHToU/v7+2LZtG06cOIFp06ahqKjoqft61OrVq3HkyBG0b98eGzZsQOPGjcv83f/999+oW7fuc9VHJHcMREQvGXt7e/Ts2RNLly5FXl7ecz3XyckJr7zyCq5cuYKGDRvqLB4eHs9dS7NmzZCUlKRTx+HDh1GjRg00btz4ubbl4+ODjIyMFwpFj3N1dcXYsWOxadMmTJo0CStWrAAAabSqpKTkic83MzMrt0+XLl1w9uxZbNy4UQo/nTt3xu7duxEXF4fOnTtLfZs2bVrmNPa4uDg0bdr0ufZ7+PBhuLm5Ydq0aWjdujUaNWqEa9eu6fTx9PTEsWPHdNqOHz9eZvs+Pj6YOnUq4uLi4OXlpXPdp8uXL+Off/6Bj49PhfURVUcMREQvoWXLluH+/fto3bo1NmzYgPPnzyM5ORlr167FhQsXYGJiUuFzQ0NDERYWhq+//hoXL17E6dOnsXr1aixatOi56xg+fDhq1aqFESNG4MyZM9i3bx+Cg4MREBAAJyen59qWj48P6tati8OHDz93HeUJCQnBH3/8gdTUVCQmJmLv3r1SCHFzc4NKpcK2bduQlZWF3Nzccrfh7u6O3Nxc7NmzB7dv35YOI3l5ecHe3h7r1q2TAlGXLl2wefNmFBQU6Jyy/umnnyIiIgLff/89UlJSsGjRImzatAmTJ0+usHZ3d3ecOnUKycnJuH37NoqLi9GwYUNcv34d69evx+XLl/HNN98gOjpa53nBwcFYuXIlIiMjkZKSgrlz5+LUqVPSqFFqaiqmTp2KI0eO4Nq1a9i5cycuXryoE84OHjyI+vXro0GDBs//phPJmZHnMBGRnvz1119i/PjxwsPDQ5iamgorKyvx5ptvin//+98iLy/vic9dt26deP3114WZmZmwtbUVnTp1kiZpP5ykfOLECZ3nVDTR91lPu38WU6ZMEUOHDtVpe9ZJ1Y/XO378eNGgQQNhbm4u6tatKwICAsTt27el9bNnzxYajUaoVKpyT7t/aOzYscLe3l467f6hQYMGCRMTE6HVaoUQDyZb29nZidatW5fZxvOcdi/Eg7PrevbsKaysrHROu//000+Fvb29sLKyEkOGDBHh4eFlJn3Pnj1bODg4CCsrKzF69GgxYcIE0bZtWyGEEBkZGeLtt98Wzs7OwszMTLi5uYkZM2bonNnn6+tb6TMYieRMJcRjB5CJiGTq1q1baN68ORISEuDm5mbscl4KPXv2hEajwZo1a57a98yZM+jevTsuXrwItVptgOqIDIdXqiaiasPJyQkrV67E9evXGYgqIT8/H99//z38/PxgYmKC//73v9i9ezd27dr1TM//66+/EBUVxTBELyWOEBERKURBQQH69u2LxMREFBYWwtPTE19++SUGDhxo7NKIjI6BiIiIiBSPZ5kRERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHiMRARERGR4jEQERERkeIxEBEREZHi/X93Z2Jo+KgNTQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g_split_tags['first_two'] = g_split_tags[0] + \":\" + g_split_tags[1]\n",
    "plt.ylabel(\"Occurences\")\n",
    "plt.xlabel(\"G error (first two tags)\")\n",
    "g_split_tags['first_two'].value_counts().plot.bar()\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}