data_similarity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "54e83917",
   "metadata": {},
   "source": [
    "# Models to try\n",
    "- kNN\n",
    "- SVM\n",
    "- Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "056a8e5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to ../../Resources...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "import nltk\n",
    "\n",
    "nltk.download('stopwords', download_dir='../../Resources')\n",
    "nltk.data.path.append('../../Resources')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f4d32e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "STOPWORDS = set(stopwords.words('english'))\n",
    "ps = PorterStemmer()\n",
    "\n",
    "def remove_stopwords(text):\n",
    "    return ' '.join([word for word in str(text).split() if word not in STOPWORDS])\n",
    "\n",
    "def data_preprocess(text, remove_punctuation=True):\n",
    "    if remove_punctuation: \n",
    "        text = re.sub(f'[{re.escape(string.punctuation)}]', '', text)\n",
    "    text = text.strip()\n",
    "    text = re.sub(r'\\n', '', text)\n",
    "    text = text.lower()\n",
    "    text = ps.stem(text)\n",
    "    text = remove_stopwords(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "20e07678",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>full_text</th>\n",
       "      <th>cohesion</th>\n",
       "      <th>syntax</th>\n",
       "      <th>vocabulary</th>\n",
       "      <th>phraseology</th>\n",
       "      <th>grammar</th>\n",
       "      <th>conventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0016926B079C</td>\n",
       "      <td>I think that students would benefit from learn...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0022683E9EA5</td>\n",
       "      <td>When a problem is a change you have to let it ...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00299B378633</td>\n",
       "      <td>Dear, Principal\\n\\nIf u change the school poli...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>003885A45F42</td>\n",
       "      <td>The best time in life is when you become yours...</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0049B1DF5CCC</td>\n",
       "      <td>Small act of kindness can impact in other peop...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3906</th>\n",
       "      <td>FFD29828A873</td>\n",
       "      <td>I believe using cellphones in class for educat...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3907</th>\n",
       "      <td>FFD9A83B0849</td>\n",
       "      <td>Working alone, students do not have to argue w...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3908</th>\n",
       "      <td>FFDC4011AC9C</td>\n",
       "      <td>\"A problem is a chance for you to do your best...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3909</th>\n",
       "      <td>FFE16D704B16</td>\n",
       "      <td>Many people disagree with Albert Schweitzer's ...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3910</th>\n",
       "      <td>FFED00D6E0BD</td>\n",
       "      <td>Do you think that failure is the main thing fo...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3911 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           text_id                                          full_text  \\\n",
       "0     0016926B079C  I think that students would benefit from learn...   \n",
       "1     0022683E9EA5  When a problem is a change you have to let it ...   \n",
       "2     00299B378633  Dear, Principal\\n\\nIf u change the school poli...   \n",
       "3     003885A45F42  The best time in life is when you become yours...   \n",
       "4     0049B1DF5CCC  Small act of kindness can impact in other peop...   \n",
       "...            ...                                                ...   \n",
       "3906  FFD29828A873  I believe using cellphones in class for educat...   \n",
       "3907  FFD9A83B0849  Working alone, students do not have to argue w...   \n",
       "3908  FFDC4011AC9C  \"A problem is a chance for you to do your best...   \n",
       "3909  FFE16D704B16  Many people disagree with Albert Schweitzer's ...   \n",
       "3910  FFED00D6E0BD  Do you think that failure is the main thing fo...   \n",
       "\n",
       "      cohesion  syntax  vocabulary  phraseology  grammar  conventions  \n",
       "0          3.5     3.5         3.0          3.0      4.0          3.0  \n",
       "1          2.5     2.5         3.0          2.0      2.0          2.5  \n",
       "2          3.0     3.5         3.0          3.0      3.0          2.5  \n",
       "3          4.5     4.5         4.5          4.5      4.0          5.0  \n",
       "4          2.5     3.0         3.0          3.0      2.5          2.5  \n",
       "...        ...     ...         ...          ...      ...          ...  \n",
       "3906       2.5     3.0         3.0          3.5      2.5          2.5  \n",
       "3907       4.0     4.0         4.0          4.0      3.5          3.0  \n",
       "3908       2.5     3.0         3.0          3.0      3.5          3.0  \n",
       "3909       4.0     4.5         4.5          4.0      4.5          4.5  \n",
       "3910       3.5     2.5         3.5          3.0      3.0          3.5  \n",
       "\n",
       "[3911 rows x 8 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orig_data = pd.read_csv('../Data/train.csv')\n",
    "orig_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f61de682",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>full_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dear Generic_Name Generic_Name, I believe that...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dear, Generic_Name Generic_Name Generic_Name M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dear Local Newspaper, Generic_Name I have foun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dear LOCATION_NAME, I know having computers ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5870</th>\n",
       "      <td>In most stories mothers and daughters are eit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5871</th>\n",
       "      <td>I never understood the meaning laughter is th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872</th>\n",
       "      <td>When you laugh, is Generic_Name out of habit, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5873</th>\n",
       "      <td>Trippin' on fen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5874</th>\n",
       "      <td>Many people believe that laughter can improve...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5875 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              full_text\n",
       "0     Dear local newspaper, I think effects computer...\n",
       "1     Dear Generic_Name Generic_Name, I believe that...\n",
       "2     Dear, Generic_Name Generic_Name Generic_Name M...\n",
       "3     Dear Local Newspaper, Generic_Name I have foun...\n",
       "4     Dear LOCATION_NAME, I know having computers ha...\n",
       "...                                                 ...\n",
       "5870   In most stories mothers and daughters are eit...\n",
       "5871   I never understood the meaning laughter is th...\n",
       "5872  When you laugh, is Generic_Name out of habit, ...\n",
       "5873                                 Trippin' on fen...\n",
       "5874   Many people believe that laughter can improve...\n",
       "\n",
       "[5875 rows x 1 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aug_data = pd.read_csv('../Data/aug_data.csv')\n",
    "aug_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "027700ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_data = orig_data[['full_text']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "db480371",
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_data.loc[:, 'full_text'] = orig_data['full_text'].apply(data_preprocess)\n",
    "aug_data.loc[:, 'full_text'] = aug_data['full_text'].apply(data_preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "13cd9e20",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure data is balanced\n",
    "aug_data = aug_data.sample(orig_data.shape[0], random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "64b25730",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((3911, 1), (3911, 1))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orig_data.shape, aug_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1a69fad5",
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_data['target'] = 0\n",
    "aug_data['target'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "99071216",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>full_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>think students would benefit learning home,bec...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>problem change let best matter happening chang...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dear, principalif u change school policy grade...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>best time life become yourself. agree greatest...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>small act kindness impact people change people...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7817</th>\n",
       "      <td>yes children reading anything thats like that....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7818</th>\n",
       "      <td>example remember fun laughter life generic_nam...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7819</th>\n",
       "      <td>many resaons taking book, movie, anything else...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7820</th>\n",
       "      <td>dear generic_name, believe computers benefitua...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7821</th>\n",
       "      <td>society today always going something someone d...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7822 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              full_text  target\n",
       "0     think students would benefit learning home,bec...       0\n",
       "1     problem change let best matter happening chang...       0\n",
       "2     dear, principalif u change school policy grade...       0\n",
       "3     best time life become yourself. agree greatest...       0\n",
       "4     small act kindness impact people change people...       0\n",
       "...                                                 ...     ...\n",
       "7817  yes children reading anything thats like that....       1\n",
       "7818  example remember fun laughter life generic_nam...       1\n",
       "7819  many resaons taking book, movie, anything else...       1\n",
       "7820  dear generic_name, believe computers benefitua...       1\n",
       "7821  society today always going something someone d...       1\n",
       "\n",
       "[7822 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.concat([orig_data, aug_data], axis=0).reset_index(drop=True)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "93cdc3e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>full_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>good behavior trying influence per swaying oth...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>libraies able remove books, movies, magizines ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>writing students work schools homes. people th...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>agree churchill statement important role failu...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dear generic_name, \"generic_name generic_name ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7817</th>\n",
       "      <td>censorship, sensitive topic. generic_name many...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7818</th>\n",
       "      <td>dear local newspaper, generic_name big agument...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7819</th>\n",
       "      <td>teenagers really complicated day's trying extr...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7820</th>\n",
       "      <td>ed censorship certain media controversial topi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7821</th>\n",
       "      <td>other_name, , newest technology brought public...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7822 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              full_text  target\n",
       "0     good behavior trying influence per swaying oth...       0\n",
       "1     libraies able remove books, movies, magizines ...       1\n",
       "2     writing students work schools homes. people th...       0\n",
       "3     agree churchill statement important role failu...       0\n",
       "4     dear generic_name, \"generic_name generic_name ...       1\n",
       "...                                                 ...     ...\n",
       "7817  censorship, sensitive topic. generic_name many...       1\n",
       "7818  dear local newspaper, generic_name big agument...       1\n",
       "7819  teenagers really complicated day's trying extr...       0\n",
       "7820  ed censorship certain media controversial topi...       1\n",
       "7821  other_name, , newest technology brought public...       1\n",
       "\n",
       "[7822 rows x 2 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# shuffle\n",
    "data = data.sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8254a9b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = data[['full_text']], data['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5f426751",
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_NAME = 'distilbert-base-uncased'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "991724d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1113e1fcad294c658397a748f8ef482e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c8e1495708c4d1b97d9ca808c08d38c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/483 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0498d6f6d2fd4b33b2091450f1e37124",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58143c63b7864ffb87f5c73373b0f773",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "3199fcff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(example):\n",
    "    return np.array(tokenizer(example['full_text'], \n",
    "                    max_length=512, \n",
    "                    padding='max_length', \n",
    "                    truncation=True, \n",
    "                    add_special_tokens=True,\n",
    "                    return_tensors=None)['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "2bd74d5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.DataFrame(list(X.apply(tokenize, axis=1).values), columns=list(range(512)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "1330bb36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>502</th>\n",
       "      <th>503</th>\n",
       "      <th>504</th>\n",
       "      <th>505</th>\n",
       "      <th>506</th>\n",
       "      <th>507</th>\n",
       "      <th>508</th>\n",
       "      <th>509</th>\n",
       "      <th>510</th>\n",
       "      <th>511</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>2204</td>\n",
       "      <td>5248</td>\n",
       "      <td>2667</td>\n",
       "      <td>3747</td>\n",
       "      <td>2566</td>\n",
       "      <td>21826</td>\n",
       "      <td>2500</td>\n",
       "      <td>1029</td>\n",
       "      <td>11082</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>5622</td>\n",
       "      <td>10024</td>\n",
       "      <td>3111</td>\n",
       "      <td>2583</td>\n",
       "      <td>6366</td>\n",
       "      <td>2808</td>\n",
       "      <td>1010</td>\n",
       "      <td>5691</td>\n",
       "      <td>1010</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>3015</td>\n",
       "      <td>2493</td>\n",
       "      <td>2147</td>\n",
       "      <td>2816</td>\n",
       "      <td>5014</td>\n",
       "      <td>1012</td>\n",
       "      <td>2111</td>\n",
       "      <td>2228</td>\n",
       "      <td>2190</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>5993</td>\n",
       "      <td>10888</td>\n",
       "      <td>4861</td>\n",
       "      <td>2590</td>\n",
       "      <td>2535</td>\n",
       "      <td>4945</td>\n",
       "      <td>3248</td>\n",
       "      <td>8463</td>\n",
       "      <td>3112</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>6203</td>\n",
       "      <td>12391</td>\n",
       "      <td>1035</td>\n",
       "      <td>2171</td>\n",
       "      <td>1010</td>\n",
       "      <td>1000</td>\n",
       "      <td>12391</td>\n",
       "      <td>1035</td>\n",
       "      <td>2171</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7817</th>\n",
       "      <td>101</td>\n",
       "      <td>15657</td>\n",
       "      <td>1010</td>\n",
       "      <td>7591</td>\n",
       "      <td>8476</td>\n",
       "      <td>1012</td>\n",
       "      <td>12391</td>\n",
       "      <td>1035</td>\n",
       "      <td>2171</td>\n",
       "      <td>2116</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7818</th>\n",
       "      <td>101</td>\n",
       "      <td>6203</td>\n",
       "      <td>2334</td>\n",
       "      <td>3780</td>\n",
       "      <td>1010</td>\n",
       "      <td>12391</td>\n",
       "      <td>1035</td>\n",
       "      <td>2171</td>\n",
       "      <td>2502</td>\n",
       "      <td>12943</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7819</th>\n",
       "      <td>101</td>\n",
       "      <td>12908</td>\n",
       "      <td>2428</td>\n",
       "      <td>8552</td>\n",
       "      <td>2154</td>\n",
       "      <td>1005</td>\n",
       "      <td>1055</td>\n",
       "      <td>2667</td>\n",
       "      <td>4469</td>\n",
       "      <td>1016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7820</th>\n",
       "      <td>101</td>\n",
       "      <td>3968</td>\n",
       "      <td>15657</td>\n",
       "      <td>3056</td>\n",
       "      <td>2865</td>\n",
       "      <td>6801</td>\n",
       "      <td>8476</td>\n",
       "      <td>1012</td>\n",
       "      <td>3251</td>\n",
       "      <td>3056</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7821</th>\n",
       "      <td>101</td>\n",
       "      <td>2060</td>\n",
       "      <td>1035</td>\n",
       "      <td>2171</td>\n",
       "      <td>1010</td>\n",
       "      <td>1010</td>\n",
       "      <td>14751</td>\n",
       "      <td>2974</td>\n",
       "      <td>2716</td>\n",
       "      <td>2270</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7822 rows × 512 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      0      1      2     3     4      5      6      7     8      9    ...  \\\n",
       "0     101   2204   5248  2667  3747   2566  21826   2500  1029  11082  ...   \n",
       "1     101   5622  10024  3111  2583   6366   2808   1010  5691   1010  ...   \n",
       "2     101   3015   2493  2147  2816   5014   1012   2111  2228   2190  ...   \n",
       "3     101   5993  10888  4861  2590   2535   4945   3248  8463   3112  ...   \n",
       "4     101   6203  12391  1035  2171   1010   1000  12391  1035   2171  ...   \n",
       "...   ...    ...    ...   ...   ...    ...    ...    ...   ...    ...  ...   \n",
       "7817  101  15657   1010  7591  8476   1012  12391   1035  2171   2116  ...   \n",
       "7818  101   6203   2334  3780  1010  12391   1035   2171  2502  12943  ...   \n",
       "7819  101  12908   2428  8552  2154   1005   1055   2667  4469   1016  ...   \n",
       "7820  101   3968  15657  3056  2865   6801   8476   1012  3251   3056  ...   \n",
       "7821  101   2060   1035  2171  1010   1010  14751   2974  2716   2270  ...   \n",
       "\n",
       "      502  503  504  505  506  507  508  509  510  511  \n",
       "0       0    0    0    0    0    0    0    0    0    0  \n",
       "1       0    0    0    0    0    0    0    0    0    0  \n",
       "2       0    0    0    0    0    0    0    0    0    0  \n",
       "3       0    0    0    0    0    0    0    0    0    0  \n",
       "4       0    0    0    0    0    0    0    0    0    0  \n",
       "...   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  \n",
       "7817    0    0    0    0    0    0    0    0    0    0  \n",
       "7818    0    0    0    0    0    0    0    0    0    0  \n",
       "7819    0    0    0    0    0    0    0    0    0    0  \n",
       "7820    0    0    0    0    0    0    0    0    0    0  \n",
       "7821    0    0    0    0    0    0    0    0    0    0  \n",
       "\n",
       "[7822 rows x 512 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "fb71462d",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "405d39a7",
   "metadata": {},
   "source": [
    "# kNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "22b804aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-8 {color: black;background-color: white;}#sk-container-id-8 pre{padding: 0;}#sk-container-id-8 div.sk-toggleable {background-color: white;}#sk-container-id-8 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-8 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-8 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-8 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-8 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-8 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-8 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-8 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-8 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-8 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-8 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-8 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-8 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-8 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-8 div.sk-item {position: relative;z-index: 1;}#sk-container-id-8 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-8 div.sk-item::before, #sk-container-id-8 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-8 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-8 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-8 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-8 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-8 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-8 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-8 div.sk-label-container {text-align: center;}#sk-container-id-8 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-8 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-8\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>KNeighborsClassifier(n_neighbors=7)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" checked><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier(n_neighbors=7)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "KNeighborsClassifier(n_neighbors=7)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=7)\n",
    "knn.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "ade16e3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5615801704105344"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31905fe6",
   "metadata": {},
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "0d56fda7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;standardscaler&#x27;, StandardScaler()),\n",
       "                (&#x27;svc&#x27;, SVC(gamma=&#x27;auto&#x27;))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;standardscaler&#x27;, StandardScaler()),\n",
       "                (&#x27;svc&#x27;, SVC(gamma=&#x27;auto&#x27;))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(gamma=&#x27;auto&#x27;)</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
       "                ('svc', SVC(gamma='auto'))])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm = make_pipeline(StandardScaler(), SVC(gamma='auto'))\n",
    "svm.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "e3ce827d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6250968241673122"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "695f929e",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "ec189e13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-10 {color: black;background-color: white;}#sk-container-id-10 pre{padding: 0;}#sk-container-id-10 div.sk-toggleable {background-color: white;}#sk-container-id-10 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-10 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-10 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-10 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-10 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-10 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-10 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-10 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-10 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-10 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-10 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-10 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-10 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-10 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-10 div.sk-item {position: relative;z-index: 1;}#sk-container-id-10 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-10 div.sk-item::before, #sk-container-id-10 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-10 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-10 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-10 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-10 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-10 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-10 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-10 div.sk-label-container {text-align: center;}#sk-container-id-10 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-10 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-10\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier(max_depth=25, random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" checked><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(max_depth=25, random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomForestClassifier(max_depth=25, random_state=42)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = RandomForestClassifier(max_depth=25, random_state=42)\n",
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "b14a1409",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7358636715724245"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55a3ed78",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}