fairness_classification/svm/svm_combined_word-loocv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5e23e5f3",
   "metadata": {},
   "source": [
    "#### Training 8 SVMs with Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "775a61fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import ssl, unicodedata, contractions\n",
    "\n",
    "try:\n",
    "    _create_unverified_https_context = ssl._create_unverified_context\n",
    "except AttributeError:\n",
    "    pass\n",
    "else:\n",
    "    ssl._create_default_https_context = _create_unverified_https_context\n",
    "nltk.download('wordnet', quiet=True)\n",
    "nltk.download('punkt', quiet=True)\n",
    "nltk.download('omw-1.4', quiet=True)\n",
    "from nltk.tokenize import word_tokenize\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import string\n",
    "import os\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import LeaveOneGroupOut\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8d775af6",
   "metadata": {},
   "outputs": [],
   "source": [
    "files = os.listdir(\"./data/Postags/\")\n",
    "prefix_sentences = \"./data/Sentences/\"\n",
    "prefix_label_A = \"./data/Labels_A/\"\n",
    "prefix_label_CH = \"./data/Labels_CH/\"\n",
    "prefix_label_CR = \"./data/Labels_CR/\"\n",
    "prefix_label_J = \"./data/Labels_J/\"\n",
    "prefix_label_LAW = \"./data/Labels_LAW/\"\n",
    "prefix_label_LTD = \"./data/Labels_LTD/\"\n",
    "prefix_label_TER = \"./data/Labels_TER/\"\n",
    "prefix_label_USE = \"./data/Labels_USE/\"\n",
    "prefix_label = \"./data/Labels/\"\n",
    "    \n",
    "word = []\n",
    "for file in files:\n",
    "    label_file_path_A = prefix_label_A + file\n",
    "    label_file_path_CH = prefix_label_CH + file\n",
    "    label_file_path_CR = prefix_label_CR + file\n",
    "    label_file_path_J = prefix_label_J + file\n",
    "    label_file_path_LAW = prefix_label_LAW + file\n",
    "    label_file_path_LTD = prefix_label_LTD + file\n",
    "    label_file_path_TER = prefix_label_TER + file\n",
    "    label_file_path_USE = prefix_label_USE + file\n",
    "    label_file_path = prefix_label + file\n",
    "    sentences_file_path = prefix_sentences + file\n",
    "    sentences_df = pd.read_csv(sentences_file_path, sep=\"dummy_separator\", header=None)\n",
    "    sentences_df.columns = [\"sentence\"]\n",
    "        \n",
    "    label_A_df = pd.read_csv(label_file_path_A, sep=\" \", header=None, names=[\"label_A\"])\n",
    "    label_A_df[\"label_A_converted\"] = np.where(label_A_df[\"label_A\"] == -1, 0, 1)\n",
    "\n",
    "    label_CH_df = pd.read_csv(label_file_path_CH, sep=\" \", header=None, names=[\"label_CH\"])\n",
    "    label_CH_df[\"label_CH_converted\"] = np.where(label_CH_df[\"label_CH\"] == -1, 0, 1)\n",
    "\n",
    "    label_CR_df = pd.read_csv(label_file_path_CR, sep=\" \", header=None, names=[\"label_CR\"])\n",
    "    label_CR_df[\"label_CR_converted\"] = np.where(label_CR_df[\"label_CR\"] == -1, 0, 1)\n",
    "\n",
    "    label_J_df = pd.read_csv(label_file_path_J, sep=\" \", header=None, names=[\"label_J\"])\n",
    "    label_J_df[\"label_J_converted\"] = np.where(label_J_df[\"label_J\"] == -1, 0, 1)\n",
    "\n",
    "    label_LAW_df = pd.read_csv(label_file_path_LAW, sep=\" \", header=None, names=[\"label_LAW\"])\n",
    "    label_LAW_df[\"label_LAW_converted\"] = np.where(label_LAW_df[\"label_LAW\"] == -1, 0, 1)\n",
    "\n",
    "    label_LTD_df = pd.read_csv(label_file_path_LTD, sep=\" \", header=None, names=[\"label_LTD\"])\n",
    "    label_LTD_df[\"label_LTD_converted\"] = np.where(label_LTD_df[\"label_LTD\"] == -1, 0, 1)\n",
    "\n",
    "    label_TER_df = pd.read_csv(label_file_path_TER, sep=\" \", header=None, names=[\"label_TER\"])\n",
    "    label_TER_df[\"label_TER_converted\"] = np.where(label_TER_df[\"label_TER\"] == -1, 0, 1)\n",
    "\n",
    "    label_USE_df = pd.read_csv(label_file_path_USE, sep=\" \", header=None, names=[\"label_USE\"])\n",
    "    label_USE_df[\"label_USE_converted\"] = np.where(label_USE_df[\"label_USE\"] == -1, 0, 1)\n",
    "\n",
    "    label_df = pd.read_csv(label_file_path, sep=\" \", header=None, names=[\"label\"])\n",
    "    label_df[\"label_converted\"] = np.where(label_df[\"label\"] == -1, 0, 1)\n",
    "\n",
    "    sentences_df[\"document\"] = file\n",
    "    df_concat = pd.concat([label_df[\"label_converted\"], label_A_df[\"label_A_converted\"], label_CH_df[\"label_CH_converted\"], label_CR_df[\"label_CR_converted\"],\n",
    "                           label_J_df[\"label_J_converted\"], label_LAW_df[\"label_LAW_converted\"], label_LTD_df[\"label_LTD_converted\"],\n",
    "                           label_TER_df[\"label_TER_converted\"], label_USE_df[\"label_USE_converted\"], sentences_df[[\"sentence\", \"document\"]]], axis=1)\n",
    "    word.append(df_concat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "41ac0192",
   "metadata": {},
   "outputs": [],
   "source": [
    "colnames = [\"label_converted\", \"label_A_converted\", \"label_CH_converted\", \"label_CR_converted\", \"label_J_converted\", \"label_LAW_converted\", \"label_LTD_converted\", \"label_TER_converted\", \"label_USE_converted\", \"sentence\", \"document\"]\n",
    "clauses_df = pd.DataFrame(columns = colnames)\n",
    "for df in word:\n",
    "    clauses_df = clauses_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bcd3cf12",
   "metadata": {},
   "outputs": [],
   "source": [
    "clauses_df.rename(columns={'label_converted': 'label', 'label_A_converted': 'label_A', 'label_CH_converted': 'label_CH', 'label_CR_converted': 'label_CR', \n",
    "                           'label_J_converted': 'label_J', 'label_LAW_converted': 'label_LAW', 'label_LTD_converted': 'label_LTD', \n",
    "                           'label_TER_converted': 'label_TER', 'label_USE_converted': 'label_USE',\n",
    "                           'sentence': 'sentences', 'document' : 'document'}, inplace=True)\n",
    "clauses_df.to_csv(\"data/svm8_word_merged.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2ded1369",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_A</th>\n",
       "      <th>label_CH</th>\n",
       "      <th>label_CR</th>\n",
       "      <th>label_J</th>\n",
       "      <th>label_LAW</th>\n",
       "      <th>label_LTD</th>\n",
       "      <th>label_TER</th>\n",
       "      <th>label_USE</th>\n",
       "      <th>sentences</th>\n",
       "      <th>document</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>thanks for sending us good vibes by using the ...</td>\n",
       "      <td>Viber.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>you may be surprised , but we will refer to al...</td>\n",
       "      <td>Viber.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>the terms of use -lrb- or , the `` terms '' -r...</td>\n",
       "      <td>Viber.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>the language of the terms will seem legal -lrb...</td>\n",
       "      <td>Viber.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>when you use our services , in addition to enj...</td>\n",
       "      <td>Viber.txt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label label_A label_CH label_CR label_J label_LAW label_LTD label_TER  \\\n",
       "0     0       0        0        0       0         0         0         0   \n",
       "1     0       0        0        0       0         0         0         0   \n",
       "2     0       0        0        0       0         0         0         0   \n",
       "3     0       0        0        0       0         0         0         0   \n",
       "4     1       0        0        0       0         0         0         0   \n",
       "\n",
       "  label_USE                                          sentences   document  \n",
       "0         0  thanks for sending us good vibes by using the ...  Viber.txt  \n",
       "1         0  you may be surprised , but we will refer to al...  Viber.txt  \n",
       "2         0  the terms of use -lrb- or , the `` terms '' -r...  Viber.txt  \n",
       "3         0  the language of the terms will seem legal -lrb...  Viber.txt  \n",
       "4         1  when you use our services , in addition to enj...  Viber.txt  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clauses_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "98e993a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (clauses_df.isnull().sum().all() == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d86eb094",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting: to_lower\n",
      "Ended: to_lower\n",
      "Starting: remove_special_words\n",
      "Ended: remove_special_words\n",
      "Starting: remove_accented_characters\n",
      "Ended: remove_accented_characters\n",
      "Starting: remove_html_encodings\n",
      "Ended: remove_html_encodings\n",
      "Starting: remove_html_tags\n",
      "Ended: remove_html_tags\n",
      "Starting: remove_url\n",
      "Ended: remove_url\n",
      "Starting: fix_contractions\n",
      "Ended: fix_contractions\n",
      "Starting: remove_non_alpha_characters\n",
      "Ended: remove_non_alpha_characters\n",
      "Starting: remove_extra_spaces\n",
      "Ended: remove_extra_spaces\n"
     ]
    }
   ],
   "source": [
    "def to_lower(data: pd.Series):\n",
    "    return data.str.lower()\n",
    "\n",
    "def remove_accented_characters(data: pd.Series):\n",
    "    return data.apply(lambda x: unicodedata.normalize(\"NFKD\", x).encode(\"ascii\", \"ignore\").decode(\"utf-8\", \"ignore\"))\n",
    "\n",
    "def remove_html_encodings(data: pd.Series):\n",
    "    return data.str.replace(r\"\\d+;\", \" \", regex=True)\n",
    "\n",
    "def remove_html_tags(data: pd.Series):\n",
    "    return data.str.replace(r\"<[a-zA-Z]+\\s?/?>\", \" \", regex=True)\n",
    "\n",
    "def remove_url(data: pd.Series):\n",
    "    return data.str.replace(r\"https?://([\\w\\-\\._]+){2,}/[\\w\\-\\.\\-/=\\+_\\?]+\", \" \", regex=True)\n",
    "\n",
    "def remove_html_and_url(data: pd.Series):\n",
    "    data.str.replace(r\"\\d+;\", \" \", regex=True)\n",
    "    data.str.replace(r\"<[a-zA-Z]+\\s?/?>\", \" \", regex=True)\n",
    "    data.str.replace(r\"https?://([\\w\\-\\._]+){2,}/[\\w\\-\\.\\-/=\\+_\\?]+\", \" \", regex=True)\n",
    "    return data\n",
    "\n",
    "def remove_extra_spaces(data: pd.Series):\n",
    "    return data.str.replace(r\"^\\s*|\\s\\s*\", \" \", regex=True)\n",
    "                     \n",
    "def remove_non_alpha_characters(data: pd.Series):\n",
    "    return data.str.replace(r\"_+|\\\\|[^a-zA-Z0-9\\s]\", \" \", regex=True)\n",
    "\n",
    "def fix_contractions(data: pd.Series):\n",
    "    def contraction_fixer(txt: str):\n",
    "        return \" \".join([contractions.fix(word) for word in txt.split()])\n",
    "    return data.apply(contraction_fixer)\n",
    "\n",
    "def remove_special_words(data: pd.Series):\n",
    "    return data.str.replace(r\"\\-[^a-zA-Z]{3}\\-\", \" \", regex=True)\n",
    "                     \n",
    "data_cleaning_pipeline = {\n",
    "    \"sentences\": [\n",
    "        to_lower,\n",
    "        remove_special_words,\n",
    "        remove_accented_characters,\n",
    "        remove_html_encodings,\n",
    "        remove_html_tags,\n",
    "        remove_url,\n",
    "        fix_contractions,\n",
    "        remove_non_alpha_characters,\n",
    "        remove_extra_spaces,\n",
    "    ]\n",
    "}\n",
    "\n",
    "cleaned_data = clauses_df.copy()\n",
    "\n",
    "for col, pipeline in data_cleaning_pipeline.items():\n",
    "    temp_data = cleaned_data[col].copy()\n",
    "    for func in pipeline:\n",
    "        print(f\"Starting: {func.__name__}\")\n",
    "        temp_data = func(temp_data)\n",
    "        print(f\"Ended: {func.__name__}\")\n",
    "    cleaned_data[col] = temp_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d199175a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Viber.txt', 'Nintendo.txt', 'Tinder.txt', 'Dropbox.txt',\n",
       "       'Microsoft.txt', 'Betterpoints_UK.txt', 'Airbnb.txt',\n",
       "       'musically.txt', 'Crowdtangle.txt', 'TripAdvisor.txt',\n",
       "       'Deliveroo.txt', 'Moves-app.txt', 'Spotify.txt', 'Supercell.txt',\n",
       "       '9gag.txt', 'Booking.txt', 'Headspace.txt', 'Fitbit.txt',\n",
       "       'Syncme.txt', 'Vimeo.txt', 'Oculus.txt', 'Endomondo.txt',\n",
       "       'Instagram.txt', 'LindenLab.txt', 'WorldOfWarcraft.txt',\n",
       "       'YouTube.txt', 'Academia.txt', 'Yahoo.txt', 'WhatsApp.txt',\n",
       "       'Google.txt', 'Zynga.txt', 'Facebook.txt', 'Amazon.txt',\n",
       "       'Vivino.txt', 'Netflix.txt', 'PokemonGo.txt', 'Skype.txt',\n",
       "       'Snap.txt', 'eBay.txt', 'Masquerade.txt', 'Twitter.txt',\n",
       "       'LinkedIn.txt', 'Skyscanner.txt', 'Duolingo.txt', 'TrueCaller.txt',\n",
       "       'Uber.txt', 'Rovio.txt', 'Atlas.txt', 'Evernote.txt', 'Onavo.txt'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_data.document.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "690d02d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>thanks for sending us good vibes by using the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>you may be surprised but we will refer to all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the terms of use lrb or the terms rrb present...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>the language of the terms will seem legal lrb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>when you use our services in addition to enjo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>the failure of onavo to enforce any right or ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>the section headings in the agreement are inc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>including whether capitalized or not means wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>this agreement may not be assigned by you wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>last updated december 20 2013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9414 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             sentences\n",
       "0     thanks for sending us good vibes by using the...\n",
       "1     you may be surprised but we will refer to all...\n",
       "2     the terms of use lrb or the terms rrb present...\n",
       "3     the language of the terms will seem legal lrb...\n",
       "4     when you use our services in addition to enjo...\n",
       "..                                                 ...\n",
       "142   the failure of onavo to enforce any right or ...\n",
       "143   the section headings in the agreement are inc...\n",
       "144   including whether capitalized or not means wi...\n",
       "145   this agreement may not be assigned by you wit...\n",
       "146                      last updated december 20 2013\n",
       "\n",
       "[9414 rows x 1 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_data[['sentences']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "443264f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logo = LeaveOneGroupOut()\n",
    "X = cleaned_data['sentences']\n",
    "y = cleaned_data[['label', 'label_A', 'label_CH', 'label_CR', 'label_J', 'label_LAW', 'label_LTD', 'label_TER', 'label_USE']]\n",
    "group = cleaned_data['document']\n",
    "logo.get_n_splits(X, y, group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e9e32e67",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_val_test = []\n",
    "for train_val_index, test_index in logo.split(X, y, group):\n",
    "    train_val, test = cleaned_data.iloc[train_val_index], cleaned_data.iloc[test_index]\n",
    "    train_val_test.append((train_val, test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "967f2bac",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_svm(x_batch, y_batch, label, nrange):\n",
    "    X_train = x_batch[\"sentences\"]\n",
    "    y_train = x_batch[label]\n",
    "    train_groups = x_batch[\"document\"]\n",
    "    X_test = y_batch[\"sentences\"]\n",
    "    y_test = y_batch[label]\n",
    "    test_document = y_batch.document.unique()[0]\n",
    "    vectorizer = TfidfVectorizer(lowercase = True, ngram_range = nrange)\n",
    "    X_train = vectorizer.fit_transform(X_train)\n",
    "    X_test = vectorizer.transform(X_test)\n",
    "    y_train = y_train.astype('int')\n",
    "    y_test = y_test.astype('int')\n",
    "    \n",
    "    svm = LinearSVC(random_state=0, max_iter = 5000)\n",
    "    Cs = [0.001, 0.01, 0.1, 1, 10]\n",
    "    clf = GridSearchCV(estimator=svm, param_grid=dict(C=Cs),n_jobs=-1, scoring = 'f1_macro', refit = True)\n",
    "    clf_fit = clf.fit(X_train, y_train, groups = train_groups)\n",
    "    clf_best = clf_fit.best_estimator_\n",
    "    y_test_pred = clf_best.predict(X_test)\n",
    "    score = clf.score(X_test, y_test)\n",
    "    return y_test_pred, score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a0ef90d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training ngram_range:  (1, 1)\n",
      "Training ngram_range:  (1, 2)\n",
      "Training ngram_range:  (2, 2)\n",
      "Training ngram_range:  (1, 3)\n",
      "Training ngram_range:  (2, 3)\n",
      "Training ngram_range:  (3, 3)\n"
     ]
    }
   ],
   "source": [
    "ngram_ranges = [(1,1), (1,2), (2,2), (1,3), (2,3), (3,3)]\n",
    "scores_compare = {}\n",
    "y_test_pred_A = {}\n",
    "y_test_pred_CH = {}\n",
    "y_test_pred_CR = {}\n",
    "y_test_pred_J = {}\n",
    "y_test_pred_LAW = {}\n",
    "y_test_pred_LTD = {}\n",
    "y_test_pred_TER = {}\n",
    "y_test_pred_USE = {}\n",
    "y_test_all = {}\n",
    "for nrange in ngram_ranges:\n",
    "    print('Training ngram_range: ', nrange)\n",
    "    scores_compare[nrange] = {}\n",
    "    scores_A = []\n",
    "    scores_CH = []\n",
    "    scores_CR = []\n",
    "    scores_J = []\n",
    "    scores_LAW = []\n",
    "    scores_LTD = []\n",
    "    scores_TER = []\n",
    "    scores_USE = []\n",
    "    y_test_pred_A[nrange] = []\n",
    "    y_test_pred_CH[nrange] = []\n",
    "    y_test_pred_CR[nrange] = []\n",
    "    y_test_pred_J[nrange] = []\n",
    "    y_test_pred_LAW[nrange] = []\n",
    "    y_test_pred_LTD[nrange] = []\n",
    "    y_test_pred_TER[nrange] = []\n",
    "    y_test_pred_USE[nrange] = []\n",
    "    y_test_all[nrange] = []\n",
    "    for batch in train_val_test:\n",
    "        ## Train on Label A\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_A', nrange)\n",
    "        y_test_pred_A[nrange].extend(y_test_pred)\n",
    "        scores_A.append(score)\n",
    "\n",
    "        ## Train on Label CH\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_CH', nrange)\n",
    "        y_test_pred_CH[nrange].extend(y_test_pred)\n",
    "        scores_CH.append(score)\n",
    "\n",
    "        ## Train on Label CR\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_CR', nrange)\n",
    "        y_test_pred_CR[nrange].extend(y_test_pred)\n",
    "        scores_CR.append(score)\n",
    "\n",
    "        ## Train on Label J\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_J', nrange)\n",
    "        y_test_pred_J[nrange].extend(y_test_pred)\n",
    "        scores_J.append(score)\n",
    "\n",
    "        ## Train on Label LAW\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_LAW', nrange)\n",
    "        y_test_pred_LAW[nrange].extend(y_test_pred)\n",
    "        scores_LAW.append(score)\n",
    "\n",
    "        ## Train on Label LTD\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_LTD', nrange)\n",
    "        y_test_pred_LTD[nrange].extend(y_test_pred)\n",
    "        scores_LTD.append(score)\n",
    "\n",
    "        ## Train on Label TER\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_TER', nrange)\n",
    "        y_test_pred_TER[nrange].extend(y_test_pred)\n",
    "        scores_TER.append(score)\n",
    "\n",
    "        ## Train on Label USE\n",
    "        y_test_pred, score = train_svm(batch[0], batch[1], 'label_USE', nrange)\n",
    "        y_test_pred_USE[nrange].extend(y_test_pred)\n",
    "        scores_USE.append(score)\n",
    "\n",
    "        ## Getting all ground truth labels\n",
    "        y_test_all[nrange].extend(batch[1][\"label\"])\n",
    "        \n",
    "    scores_compare[nrange]['A'] = sum(scores_A)/len(scores_A)\n",
    "    scores_compare[nrange]['CH'] = sum(scores_CH)/len(scores_CH)\n",
    "    scores_compare[nrange]['CR'] = sum(scores_CR)/len(scores_CR)\n",
    "    scores_compare[nrange]['J'] = sum(scores_J)/len(scores_J)\n",
    "    scores_compare[nrange]['LAW'] = sum(scores_LAW)/len(scores_LAW)\n",
    "    scores_compare[nrange]['LTD'] = sum(scores_LTD)/len(scores_LTD)\n",
    "    scores_compare[nrange]['TER'] = sum(scores_TER)/len(scores_TER)\n",
    "    scores_compare[nrange]['USE'] = sum(scores_USE)/len(scores_USE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7ab4c847",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(1, 1): {'A': 0.8425046847114963,\n",
       "  'CH': 0.8246837446214763,\n",
       "  'CR': 0.7935347576420055,\n",
       "  'J': 0.9122995996705555,\n",
       "  'LAW': 0.9067203368180956,\n",
       "  'LTD': 0.8243926197913812,\n",
       "  'TER': 0.8328813937223555,\n",
       "  'USE': 0.8750391589237424},\n",
       " (1, 2): {'A': 0.8702003455597676,\n",
       "  'CH': 0.8435952160238588,\n",
       "  'CR': 0.7982276024498073,\n",
       "  'J': 0.8877498111766744,\n",
       "  'LAW': 0.9133956363712639,\n",
       "  'LTD': 0.848403736165627,\n",
       "  'TER': 0.8331598275287863,\n",
       "  'USE': 0.894246203413507},\n",
       " (2, 2): {'A': 0.8467703451613814,\n",
       "  'CH': 0.8267391174543128,\n",
       "  'CR': 0.7271591428774837,\n",
       "  'J': 0.8775274300178688,\n",
       "  'LAW': 0.9177720803485865,\n",
       "  'LTD': 0.8580239986769304,\n",
       "  'TER': 0.8008437425891344,\n",
       "  'USE': 0.8659264004678923},\n",
       " (1, 3): {'A': 0.8668501164190343,\n",
       "  'CH': 0.8357288255863026,\n",
       "  'CR': 0.779332439877307,\n",
       "  'J': 0.9122949177196145,\n",
       "  'LAW': 0.9067456770777875,\n",
       "  'LTD': 0.8605759054744407,\n",
       "  'TER': 0.8291293158472912,\n",
       "  'USE': 0.8961195628950361},\n",
       " (2, 3): {'A': 0.8516279333271535,\n",
       "  'CH': 0.8215702199810276,\n",
       "  'CR': 0.7101927379406956,\n",
       "  'J': 0.8966211861741215,\n",
       "  'LAW': 0.9044137163439694,\n",
       "  'LTD': 0.854347570412671,\n",
       "  'TER': 0.7993947937005089,\n",
       "  'USE': 0.872398664020085},\n",
       " (3, 3): {'A': 0.8449478077592423,\n",
       "  'CH': 0.7793854079916441,\n",
       "  'CR': 0.6727676791613859,\n",
       "  'J': 0.8928958126493302,\n",
       "  'LAW': 0.9147360511089566,\n",
       "  'LTD': 0.8376475696432609,\n",
       "  'TER': 0.7663898319708674,\n",
       "  'USE': 0.8663859173836395}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_compare"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78bfe57c",
   "metadata": {},
   "source": [
    "### Combine predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "db38a0f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = {}\n",
    "for nrange in ngram_ranges:\n",
    "    y_pred[nrange] = []\n",
    "    for idx in range(len(y_test_all[nrange])):\n",
    "        if y_test_pred_A[nrange][idx] == 1 or y_test_pred_CH[nrange][idx] == 1 or y_test_pred_CR[nrange][idx] == 1 or y_test_pred_J[nrange][idx] == 1 or y_test_pred_LTD[nrange][idx] == 1 or y_test_pred_TER[nrange][idx] == 1 or y_test_pred_USE[nrange][idx] == 1:\n",
    "            y_pred[nrange].append(1) ## if anyone model predicts clause as unfair then it is unfair\n",
    "        else:\n",
    "            y_pred[nrange].append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "339ef20f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ngram range:  (1, 1)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      8382\n",
      "           1       0.83      0.60      0.70      1032\n",
      "\n",
      "    accuracy                           0.94      9414\n",
      "   macro avg       0.89      0.79      0.83      9414\n",
      "weighted avg       0.94      0.94      0.94      9414\n",
      "\n",
      "Ngram range:  (1, 2)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      8382\n",
      "           1       0.86      0.62      0.72      1032\n",
      "\n",
      "    accuracy                           0.95      9414\n",
      "   macro avg       0.91      0.80      0.84      9414\n",
      "weighted avg       0.94      0.95      0.94      9414\n",
      "\n",
      "Ngram range:  (2, 2)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      8382\n",
      "           1       0.86      0.58      0.69      1032\n",
      "\n",
      "    accuracy                           0.94      9414\n",
      "   macro avg       0.91      0.78      0.83      9414\n",
      "weighted avg       0.94      0.94      0.94      9414\n",
      "\n",
      "Ngram range:  (1, 3)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.97      8382\n",
      "           1       0.86      0.63      0.73      1032\n",
      "\n",
      "    accuracy                           0.95      9414\n",
      "   macro avg       0.91      0.81      0.85      9414\n",
      "weighted avg       0.95      0.95      0.95      9414\n",
      "\n",
      "Ngram range:  (2, 3)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      8382\n",
      "           1       0.86      0.58      0.70      1032\n",
      "\n",
      "    accuracy                           0.94      9414\n",
      "   macro avg       0.91      0.79      0.83      9414\n",
      "weighted avg       0.94      0.94      0.94      9414\n",
      "\n",
      "Ngram range:  (3, 3)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      8382\n",
      "           1       0.83      0.54      0.65      1032\n",
      "\n",
      "    accuracy                           0.94      9414\n",
      "   macro avg       0.89      0.76      0.81      9414\n",
      "weighted avg       0.93      0.94      0.93      9414\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for nrange in ngram_ranges:    \n",
    "    print('Ngram range: ', nrange)\n",
    "    report = classification_report(y_test_all[nrange], y_pred[nrange])\n",
    "    print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a67b8e2",
   "metadata": {},
   "source": [
    "ngram range of (1, 2) i.e. unigrams and bigrams gives the best macro average F1-score for combined SVM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "172a1f82",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}