Stance Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stance Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import nltk\n",
    "import numpy as np\n",
    "import preprocessing, feature_engineering, helpers\n",
    "import importlib\n",
    "from collections import Counter\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import score\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import time\n",
    "import itertools\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "importlib.reload(preprocessing)\n",
    "importlib.reload(feature_engineering)\n",
    "importlib.reload(helpers)\n",
    "importlib.reload(utils)\n",
    "preprocess = preprocessing.Preprocessing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13427, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>Body ID</th>\n",
       "      <th>Stance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hundreds of Palestinians flee floods in Gaza a...</td>\n",
       "      <td>158</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Spider burrowed through tourist's stomach and ...</td>\n",
       "      <td>1923</td>\n",
       "      <td>disagree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>'Nasa Confirms Earth Will Experience 6 Days of...</td>\n",
       "      <td>154</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Banksy 'Arrested &amp; Real Identity Revealed' Is ...</td>\n",
       "      <td>1739</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Gateway Pundit</td>\n",
       "      <td>2327</td>\n",
       "      <td>discuss</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Headline  Body ID    Stance\n",
       "1   Hundreds of Palestinians flee floods in Gaza a...      158     agree\n",
       "4   Spider burrowed through tourist's stomach and ...     1923  disagree\n",
       "5   'Nasa Confirms Earth Will Experience 6 Days of...      154     agree\n",
       "8   Banksy 'Arrested & Real Identity Revealed' Is ...     1739     agree\n",
       "10                                     Gateway Pundit     2327   discuss"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_stances = pd.read_csv(\"fn_data/train_stances.csv\")\n",
    "train_stances = train_stances.loc[lambda x: x.Stance != \"unrelated\"]\n",
    "print(train_stances.shape)\n",
    "train_stances.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1683, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Body ID</th>\n",
       "      <th>articleBody</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Last week we hinted at what was to come as Ebo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>(NEWSER) – Wonder how long a Quarter Pounder w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Posting photos of a gun-toting child online, I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>At least 25 suspected Boko Haram insurgents we...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Body ID                                        articleBody\n",
       "0        0  A small meteorite crashed into a wooded area i...\n",
       "1        4  Last week we hinted at what was to come as Ebo...\n",
       "2        5  (NEWSER) – Wonder how long a Quarter Pounder w...\n",
       "3        6  Posting photos of a gun-toting child online, I...\n",
       "4        7  At least 25 suspected Boko Haram insurgents we..."
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_bodies = pd.read_csv(\"fn_data/train_bodies.csv\")\n",
    "print(train_bodies.shape)\n",
    "train_bodies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10743, 3), (2684, 3))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stances_tr, stances_val = preprocess.train_test_split(train_bodies, train_stances)\n",
    "stances_tr.shape, stances_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'discuss': 1794, 'agree': 718, 'disagree': 172})\n",
      "0.6684053651266766\n"
     ]
    }
   ],
   "source": [
    "ct = Counter(stances_val['Stance'])\n",
    "print(ct)\n",
    "print(ct.most_common(1)[0][1]/len(list(stances_val[\"Stance\"])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "glove_dict = preprocess.get_glove_dict(\"glove.6B.50d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[([('banksy', 'NN')], {'compound': 0.0, 'neg': 0.0, 'neu': 1.0, 'pos': 0.0}),\n",
       " ([('arrested', 'VBN')],\n",
       "  {'compound': -0.4767, 'neg': 1.0, 'neu': 0.0, 'pos': 0.0}),\n",
       " ([('real', 'JJ')], {'compound': 0.0, 'neg': 0.0, 'neu': 1.0, 'pos': 0.0}),\n",
       " ([('identity', 'NN')], {'compound': 0.0, 'neg': 0.0, 'neu': 1.0, 'pos': 0.0}),\n",
       " ([('revealed', 'VBD')],\n",
       "  {'compound': 0.0, 'neg': 0.0, 'neu': 1.0, 'pos': 0.0}),\n",
       " ([('hoax', 'NN')], {'compound': -0.2732, 'neg': 1.0, 'neu': 0.0, 'pos': 0.0}),\n",
       " ([('last', 'JJ')], {'compound': 0.0, 'neg': 0.0, 'neu': 1.0, 'pos': 0.0}),\n",
       " ([('year', 'NN')], {'compound': 0.0, 'neg': 0.0, 'neu': 1.0, 'pos': 0.0})]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(nltk.pos_tag([x]),preprocess.get_sentiment(x)) for x in preprocess.get_clean_tokens(list(stances_tr.iloc[2,:])[0], False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7902431610341372"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess.cosine_similarity(glove_dict['reveal'], glove_dict['revealed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed 100\n",
      "processed 200\n",
      "processed 300\n",
      "processed 400\n",
      "processed 500\n",
      "processed 600\n",
      "processed 700\n",
      "processed 800\n",
      "processed 900\n",
      "processed 1000\n",
      "processed 1100\n",
      "processed 1200\n",
      "processed 1300\n",
      "processed 1400\n",
      "processed 1500\n",
      "processed 1600\n",
      "done! processed 1683\n"
     ]
    }
   ],
   "source": [
    "body_dict = preprocess.process_bodies_stance(train_bodies, glove_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "train_feats = [preprocess.process_feats_stance(i, body_dict, glove_dict) for i in stances_tr.values]\n",
    "val_feats = [preprocess.process_feats_stance(i, body_dict, glove_dict) for i in stances_val.values]\n",
    "end = time.time()\n",
    "print(int(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batch(data, targets, i,batch_size):\n",
    "    batches = data[i*batch_size:i*batch_size+batch_size]\n",
    "    results = targets[i*batch_size:i*batch_size+batch_size]\n",
    "    results = [(2 if result == \"agree\" else (1 if result == \"discuss\" else 0)) for result in results]\n",
    "    return np.array(batches),np.array(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Test the Model\n",
    "def eval_model(model):\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    batch_x_test,batch_y_test = get_batch(val_feats,[str(x[-1]) for x in stances_val.values],0,len(stances_val))\n",
    "    model.eval()\n",
    "    predicted = None\n",
    "    with torch.no_grad():\n",
    "        inputs = Variable(torch.FloatTensor(batch_x_test))\n",
    "        labels = torch.LongTensor(batch_y_test)\n",
    "        outputs = model(inputs)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum()\n",
    "\n",
    "    print('Accuracy: %d %%' % (100 * correct / total))\n",
    "    return predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score_model(predictions):    \n",
    "    #use FNC scorer to generate score report\n",
    "    label_prediction = [(\"agree\" if x == 2 else (\"discuss\" if x == 1 else \"disagree\")) for x in predictions]\n",
    "    label_actual = pd.DataFrame(stances_val)['Stance']\n",
    "    matrix = confusion_matrix(label_actual,label_prediction)\n",
    "    print('confusion matrix: \\n{}\\n'.format(matrix))\n",
    "    score.report_score(label_actual, label_prediction)\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def binary_accuracy(preds, y):\n",
    "    \"\"\"\n",
    "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
    "    \"\"\"\n",
    "    correct = (preds == y).float() #convert into float for division \n",
    "    acc = correct.sum()/len(correct)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#architecture from https://github.com/bentrevett/pytorch-sentiment-analysis/blob/master/4%20-%20Convolutional%20Sentiment%20Analysis.ipynb\n",
    "\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self, embedding_dim, n_filters, filter_sizes, output_dim, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.convs = nn.ModuleList([nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(fs,embedding_dim)) for fs in filter_sizes])\n",
    "        self.fc = nn.Linear(len(filter_sizes)*n_filters, output_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, x): \n",
    "        #x (batch size, 60, 91)\n",
    "        embedded = x.unsqueeze(1) \n",
    "        #embedded (batch size, 1, 60, 91)\n",
    "        conved = [F.relu(conv(embedded)).squeeze(3) for conv in self.convs] \n",
    "        #conv_n = [batch size, n_filters, 60 - filter_sizes[n]]\n",
    "        pooled = [F.max_pool1d(conv, conv.shape[2]).squeeze(2) for conv in conved] \n",
    "        #pooled_n = [batch size, n_filters]\n",
    "        cat = self.dropout(torch.cat(pooled, dim=1)) \n",
    "        #cat = [batch size, n_filters * len(filter_sizes)]\n",
    "        return self.fc(cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 92\n",
    "N_FILTERS = 50\n",
    "FILTER_SIZES = [3,4,5,8,10]\n",
    "OUTPUT_DIM = 3\n",
    "DROPOUT = 0.5\n",
    "# Parameters\n",
    "num_epochs = 5\n",
    "batch_size = 250\n",
    "display_step = 1\n",
    "\n",
    "model = CNN(EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-5)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(model, total_batch, optimizer, criterion):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    model.train()\n",
    "    for i in range(total_batch):\n",
    "        batch_x,batch_y = get_batch(train_feats, [str(x[-1]) for x in stances_tr.values],i,batch_size)\n",
    "        inputs = Variable(torch.FloatTensor(batch_x))\n",
    "        labels = Variable(torch.LongTensor(batch_y))\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        predictions = model(inputs)\n",
    "        loss = criterion(predictions, labels)\n",
    "        _, predicted = torch.max(predictions.data, 1)\n",
    "        acc = binary_accuracy(predicted, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "    return epoch_loss / total_batch, epoch_acc / total_batch\n",
    "\n",
    "def evaluate(model, total_batch, criterion):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for i in range(total_batch):\n",
    "            batch_x,batch_y = get_batch(val_feats, [str(x[-1]) for x in stances_val.values],i,batch_size)\n",
    "            inputs = Variable(torch.FloatTensor(batch_x))\n",
    "            labels = Variable(torch.LongTensor(batch_y))\n",
    "            predictions = model(inputs)\n",
    "            loss = criterion(predictions, labels)\n",
    "            _, predicted = torch.max(predictions.data, 1)\n",
    "            acc = binary_accuracy(predicted, labels)\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "    return epoch_loss / total_batch, epoch_acc / total_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Epoch: 01 | Train Loss: 0.902 | Train Acc: 56.69% | Val. Loss: 0.768 | Val. Acc: 66.92% |\n",
      "| Epoch: 02 | Train Loss: 0.768 | Train Acc: 66.64% | Val. Loss: 0.756 | Val. Acc: 67.56% |\n",
      "| Epoch: 03 | Train Loss: 0.740 | Train Acc: 68.32% | Val. Loss: 0.748 | Val. Acc: 69.00% |\n",
      "| Epoch: 04 | Train Loss: 0.719 | Train Acc: 69.10% | Val. Loss: 0.741 | Val. Acc: 70.60% |\n",
      "| Epoch: 05 | Train Loss: 0.697 | Train Acc: 70.60% | Val. Loss: 0.735 | Val. Acc: 70.36% |\n"
     ]
    }
   ],
   "source": [
    "batches_train= int(len(train_feats)/batch_size)\n",
    "batches_val = int(len(val_feats)/batch_size)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "\n",
    "    train_loss, train_acc = train(model, batches_train, optimizer, criterion)\n",
    "    valid_loss, valid_acc = evaluate(model, batches_val, criterion)\n",
    "    \n",
    "    print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}% | Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}% |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 70 %\n"
     ]
    }
   ],
   "source": [
    "# Test the Model\n",
    "predicted = eval_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 2431, 2: 253})"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_l = [i.item() for i in list(predicted)]\n",
    "Counter(predicted_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix: \n",
      "[[ 165    0  553]\n",
      " [  17    0  155]\n",
      " [  71    0 1723]]\n",
      "\n",
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |    165    |     0     |    553    |     0     |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    17     |     0     |    155    |     0     |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    71     |     0     |   1723    |     0     |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |     0     |     0     |     0     |     0     |\n",
      "-------------------------------------------------------------\n",
      "Score: 2087.0 out of 2684.0\t(77.75707898658719%)\n",
      "Normalized confusion matrix\n",
      "[[0.22980501 0.         0.77019499]\n",
      " [0.09883721 0.         0.90116279]\n",
      " [0.03957637 0.         0.96042363]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEYCAYAAAApuP8NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XecFEX6x/HPd3fJOYnsLkhSkoFsuENRUUGSCROiCOrp\nKerd6amnp545p9ML/E7PhIIoShAEEwbUAyQKCJJhFySIKJJkeX5/dC/MLhsGWGZmmefta17OdFdX\nV88uz1ZVV1fJzHDOuWSWEu8COOdcvHkgdM4lPQ+Ezrmk54HQOZf0PBA655KeB0LnXNLzQJhEJN0t\n6dXwfQNJmySllvA5lkrqUpJ5RnHOayR9H15Prf3IZ5OkxiVZtniRNEdS53iXo7TwQFiCwiCwRlKl\niG1XSJoYx2IVyMyWm1llM8uJd1n2h6QywBPA6eH1rN/XvMLjF5dc6UqepBcl3VdcOjNrZWYTY1Ck\ng4IHwpKXCtywv5ko4D+f4tUFygNz4l2QRCApLd5lKI38H1rJexS4SVL1gnZKOkHSFEkbw/+fELFv\noqT7JU0CNgONw233SfoibLqNllRL0hBJP4V5NIzI42lJK8J9X0vqVEg5GkoySWmSjg/zzn1tlbQ0\nTJci6VZJiyStl/SGpJoR+fSTtCzcd3tRX4ykCpIeD9NvlPS5pArhvl5hc+7H8JpbRBy3VNJNkmaF\nxw2TVF7SEcD8MNmPkj6KvK583+sV4fumkj4J81knaVhEOpPUNHxfTdLLktaG5b0j9w+TpP5h2R+T\ntEHSEkndirjupZJuDsv/i6TnJdWVNE7Sz5I+kFQjIv1wSavDMn4qqVW4/SqgL/Dn3N+FiPxvkTQL\n+CX8me7qopA0VtLjEfkPlfRCUT+rpGNm/iqhF7AU6AKMAO4Lt10BTAzf1wQ2AP2ANOCi8HOtcP9E\nYDnQKtxfJty2EGgCVAPmAgvC86QBLwP/jSjDJUCtcN+fgNVA+XDf3cCr4fuGgAFp+a6hDPAJ8GD4\n+QbgKyATKAf8G3g93NcS2AScGO57AtgBdCnk+3kuvJ4MgprzCeFxRwC/AKeF5/9zeM1lI77XyUB6\n+B3OA64u6DoKuq7wnFeE718HbieoBJQHfhuRzoCm4fuXgZFAlTDPBcDAcF9/4FfgyvA6rgGyARXx\ne/EVQe01A1gDTAPahGX4CLgrIv2A8LzlgKeAGRH7XiT83cqX/wygPlAh8ncxfH9oeM5TCALpYqBK\nvP+9JNIr7gU4mF7sDoRHAhuBOuQNhP2AyfmO+RLoH76fCNyTb/9E4PaIz48D4yI+94z8h1JAmTYA\nx4Tv76b4QPhPYAyQEn6eB5wasb9eGATSgDuBoRH7KgHbKSAQhoFnS25Z8u37K/BGvrRZQOeI7/WS\niP2PAP8q6DoKui7yBsKXgcFAZgHlMKApQXDbDrSM2Pe7iJ9jf2BhxL6K4bGHFvF70Tfi81vAPyM+\nDwLeKeTY6mHe1cLPL1JwIBxQ0O9ixOdzgRXAOiKCv7+ClzeNDwAz+4YgmNyab1c6sCzftmUEtYRc\nKwrI8vuI91sK+Fw590PYhJwXNqt+JKhF1o6m3JJ+B3QGLjazneHmw4C3wybrjwSBMYegdpMeWV4z\n+wUo7GZFbYLaz6IC9uX5XsJzryDv97I64v1mIq55L/0ZEDA5bIoPKKSsZcj7s8r/c9pVHjPbHL4t\nqkxR/QwlpUp6KOyK+IkgoOWWqSgF/d5EGk0Q4Oeb2efFpE06HggPnLsImk6R/3iyCQJLpAYEtZ9c\n+zwdUNgf+GfgfKCGmVUnqJkqymPvBXqb2U8Ru1YA3cysesSrvJllAasImmO5eVQkaJYXZB2wlaCJ\nn1+e70WSwnyzCkhbnF/C/1eM2HZo7hszW21mV5pZOkEt7x+5/YL5yvoreX9W+X9OB8rFQG+ClkU1\nghou7P4ZFvb7Udzvzf0Ef8TqSbpoP8t40PFAeICY2UJgGHB9xOaxwBGSLg47tC8g6GcbU0KnrULQ\nR7cWSJN0J1C1uIMk1QfeAC41swX5dv8LuF/SYWHaOpJ6h/veBHpI+q2kssA9FPI7FdbyXgCekJQe\n1nyOl1QuPHd3SacqGA7zJ2Ab8MVeXX1wnrUEAeuS8BwDiAi+kvpIygw/biAIIDvz5ZETlul+SVXC\na/8j8OrelmcfVCG49vUEwfyBfPu/B/ZqrKOkE4HLgUuBy4C/S8oo+qjk4oHwwLqHoN8MAAvGuPUg\n+Ie+nqD21sPM1pXQ+cYD7xF07C8jqIEV12QCOJWgqfumdt85zh2O8jQwCpgg6WeCTv9jw+uZA1wL\nvEZQO9wArCziPDcBs4EpwA/AwwR9kfMJbvL8naA21hPoaWbbo7zu/K4Ebib4jluRN6B2AP4naVN4\nXTdYwWMHBxHULhcDn4fXGIs7rS8T/OyyCG6MfZVv//NAy7Cr4p3iMpNUNczzOjPLMrPPwjz+G9a8\nHeFdLuecS2ZeI3TOJT0PhM65pOeB0DmX9DwQOueSnj+gHYXqNWtZvYwG8S5G3FQsW6IzdZUqs5Zv\niHcR4mrHusXrzKxOSeSVWvUwsx1bik1nW9aON7OuJXHOaHkgjEK9jAa8+M7H8S5G3BxzWIHzRySF\nw64eHu8ixNWa58/P/yTUPrMdWyjX7Pxi022d8VxUT0KVJA+EzrnYkCAlMVsXHgidc7GToFNseiB0\nzsVOgj7M4oHQORcj3jR2ziU74U1j51yykzeNnXPOm8bOuSQnbxo755Kc8Kaxcy7ZCVISM+QkZqmc\ncwenFK8ROueSmQ+fcc45H1DtnHN+s8Q557xp7JxLbj4Nl3PO4U1j51yy8ydLnHPJTnjT2DmX7LxG\n6JxzXiN0zjm/WeKcS25K3KZxYpbqIPblJx9w/mkdOO+Utrz8ryf32P/a889x4RnH0bf7b7iuX29W\nZS0HYFXWci7tdRL9enbioq7HM+K1F3Yd8+03M+h75gmcd0pbHr/nFswsZteztyaMf4+jWzWjVfOm\nPPrIQ3vs37ZtG5dcfAGtmjel0wnHsmzp0l37Hn34QVo1b8rRrZrx/oTxUeeZKE5uVZdJ93Xlqwe6\nMahbsz3233PBMXx452l8eOdpfHFfVxY80xuA3zSrs2v7h3eexrJ/nkO31ukANKhdkXF/OYWvHujG\n4N8dR5nUxKxx5VJKSrGvePAaYQzl5OTw2N0388xLb3PIoelcfs4pdDq1G40Ob74rTbOWR/PiOx9R\nvkJF3hryPM8+fDf3P/MCtescyn+GT6BsuXJs/mUTF595Ap1O7UaduvV45M4/cdv9T9OqdXv+MLAP\nX376ASecdFocr7RgOTk53Hj9tbw77n0yMjP57XEd6NGjFy1attyV5sUXnqdG9RrM+XYhbwwbyu1/\nuYVXXxvGvLlzGT5sKNNmzmFVdjZndu3C7LkLAIrNMxGkCB7q25bzn/iU7A2bGX9HF8bPyGbBqp93\npblz2Mxd7wee0pSjGlQHYNL8tZx6z/sAVK9Uhq8eOJOJc78H4I5zj+bf73/HO1NW8Mglbbm4UyNe\nmrg4hlcWvWA6wsQM1F4jjKG5M78m87DGZDRoSJmyZTmt+zl8+sHYPGnaHd+J8hUqAnBk6w6sWZ0F\nQJmyZSlbrhwAv27fju3cCcC6Nav5ZdPPHNmmA5I48+wL+fT9d2N4VdGbMnkyTZo0pVHjxpQtW5Y+\nF1zImNEj86QZM3okfftdBsA5557HxI8+xMwYM3okfS64kHLlytGwUSOaNGnKlMmTo8ozEbRtVJMl\nazaxbN0v/JpjvDN5BV1bZxSa/uyO9Rkxefke23u2y+Sj2avYsj0HgN82P4TRX68E4I0vltKtiDzj\nTlG+4sADYQyt/X4Vh9Tb/Yt6yKHprP1+VaHpRw9/heMjanbfZ6+kb/ff0KvTkfS76gbq1K3H2u9X\nUefQ9KjzjKfs7CwyM+vv+pyRkUlWVtaeaeoHadLS0qharRrr168nK2vPY7Ozs6LKMxEcWqMC2Rs2\n7/qcvWEzh9aoUGDazJoVaVC7Ep/PW7PHvrM6NODtySsAqFm5LD9t+ZWcnRbmuYV6heSZGERKSkqx\nr3hI2EAo6SxJJql58akPPuPeGca82TO45IpBu7bVTc9kyLuTePPDrxn79lDWr9vzH4or/c7qWJ8x\nX69kZ76u3kOqlad5ZjU+nrM6PgUrAZKKfUWZT1dJ8yUtlHRrAfsbSPpY0nRJsySdWVR+CRsIgYuA\nz8P/R01SYg5UAurUrceaVbtrK2tWZ1Onbr090k2eNJEX//kEjw5+bVdzOH8+jY9owcwpXwa1wtXZ\nxeaZCNLTM1i5csWuz1lZK8nIyNgzzYogzY4dO/hp40Zq1apFRsaex6anZ0SVZyJYvWEL6TUq7vqc\nXqMiqzdsKTDtWR3rM2Lyij22926fybhpWezICSLkD5u2U7VCGVLDWZ/Ta1RgVSF5JoqSCIThv/Hn\ngG5AS+AiSfk7he8A3jCzNsCFwD+KyjMhA6GkysBvgYEEF4GkFEn/kPStpPcljZV0XrhvqaSHJU0D\n+khqIuk9SV9L+iy3VimpjqS3JE0JX7+J5XW1OLotK5YtInvFMn7dvp333x1Bp1O75Ukzf84sHr7j\nDzz679eoWavOru1rVmWxdWvwS/7Txh+ZOfUrGjRuSu1DDqVS5Sp8M30KZsbYt4dyYpci//jFTfsO\nHVi48DuWLlnC9u3bGT5sKN179MqTpnuPXgx55SUARrz1JiedfAqS6N6jF8OHDWXbtm0sXbKEhQu/\no0PHjlHlmQimL91A47qVaVC7ImVSxVkd6zN+ZvYe6ZoeWoVqFcsyddH6Pfad3bEBb+frN5w0fw09\n22UCcP4JDXlvxp55JgpJKKX4VxQ6AgvNbLGZbQeGAr3zpTGgavi+GlDkF5Ood417A++Z2QJJ6yW1\nAxoBDQn+AhwCzANeiDhmvZm1BZD0IXC1mX0n6ViCvwanAE8DT5rZ55IaAOOBFgUVQNJVwFUAh6Zn\nlshFpaWlcdNdj3DD5eeyMyeHHn360viIFgx+6gGaH9maE7ucyd8fvpPNm3/h9kH9AahbL5PHBr/O\nkkULeObBO5CEmdH3iuto2qwVADf/7THu/fPv2bZ1K8ef1CVPv2IiSUtL48mnn6Vn9zPIycnhsv4D\naNmqFffcfSdt27WnR89e9B8wkAH9+9GqeVNq1KjJK0OGAtCyVSvO7XM+bY5uSVpaGk898xypqUHl\nv6A8E03OTuO216Yz9MYTSU0Rr09awvzsn/hz71bMXPoD42cG/bpndazPyCl71gbr16pIes2KfLFg\nbZ7t9705m3//7jhuPftIZi/fwGufL4nJ9eyrKJu+tSVNjfg82MwGR3zOACK/pJXAsfnyuBuYIGkQ\nUAnoUmS5EnHMmaQxwNNm9r6k64EGBEF7ppn9N0wzAnjNzN6UtBQ4ycyWhbXJtcD8iCzLmVkLSWvI\n+5ehDtDMzDYVVZ4WR7WxF9/5uMSur7Q55rDq8S5C3Bx29fB4FyGu1jx//tdm1r4k8kqr1diqnnlf\nsek2vNq3yHOGLcGuZnZF+LkfcKyZXReR5o8E8e1xSccDzwNHmtnOAsu2d5dy4EmqSVB7O0qSAakE\n1dy3izn0l/D/KcCPZta6gDQpwHFmtrWkyuuci5KItulbnCygfsTnzHBbpIFAVwAz+1JSeaA2UOAd\nxkTsIzwPeMXMDjOzhmZWH1gC/ACcG/YV1gU6F3Swmf0ELJHUB0CBY8LdE4Bdt2ElFRQsnXMHSAnd\nNZ4CHC6pkaSyBPcRRuVLsxw4NTxnC6A8QUuxQIkYCC9iz9rfW8ChBH0Bc4FXgWnAxkLy6AsMlDQT\nmMPujtTrgfbh7fS5wNUlXHbnXCFE8UEwmkBoZjuA6wj6+OcR3B2eI+keSbl3yv4EXBnGgNeB/lZE\nP2DCNY3N7OQCtj0Dwd1kM9skqRYwGZgd7m+YL/0Swmpxvu3rgAsOQLGdc1EooaYxZjYWGJtv250R\n7+cCUY8KSbhAWIwxkqoDZYF7zaz0jix1LtkocZ81LlWB0Mw6x7sMzrl954HQOZfUFD5rnIg8EDrn\nYicxK4QeCJ1zMeJ9hM45hzeNnXPOm8bOuaTnTWPnXFKT/K6xc855jdA557yP0DmX3OR3jZ1zSS5Y\n1zjepSiYB0LnXIxEv0pdrHkgdM7FTEoJTcNV0jwQOudiQ940ds4lOeE1Quec80DonEty3jR2ziW7\nYPhMYkZCD4TOuRiRN42dc85rhM65pCb5zRLnnPObJc45501j51xy86Zx6VYmNYX0GhXiXQwXBz8t\n/DbeRTho+Owzzjnns88455w3jZ1zyc4fsXPOJbtEfsQuMRcQcM4dlFJSVOwrGpK6SpovaaGkWwtJ\nc76kuZLmSHqtqPy8Ruici5mSqBFKSgWeA04DVgJTJI0ys7kRaQ4HbgN+Y2YbJB1SVJ5eI3TOxUbY\nR1jcKwodgYVmttjMtgNDgd750lwJPGdmGwDMbE1RGXogdM7FhCi+WRw2jWtLmhrxuipfVhnAiojP\nK8NtkY4AjpA0SdJXkroWVbZCm8aSqhZ1oJn9VNR+55zLLyW6Kt86M2u/n6dKAw4HOgOZwKeSjjKz\nHwtLXJg5gJF3bfrczwY02M+COueSTAndNM4C6kd8zgy3RVoJ/M/MfgWWSFpAEBinFJRhoYHQzOoX\nts855/aWBKklM6B6CnC4pEYEAfBC4OJ8ad4BLgL+K6k2QVN5cWEZRtVHKOlCSX8J32dKarcPhXfO\nJTlJxb6KY2Y7gOuA8cA84A0zmyPpHkm9wmTjgfWS5gIfAzeb2frC8ix2+IykZ4EywInAA8Bm4F9A\nh2JL7JxzEUpqPLWZjQXG5tt2Z8R7A/4YvooVzTjCE8ysraTp4Ql+kFQ2+iI751xwcyE1QZ8siSYQ\n/iopheAGCZJqATsPaKmccwefKJu+8RBNH+FzwFtAHUl/Az4HHj6gpXLOHZRKaEB1iSu2RmhmL0v6\nGugSbupjZt8c2GI55w42osTuGpe4aJ81TgV+JWge+9Mozrl9UmqbxpJuB14H0gkGLr4m6bYDXTDn\n3MElmmZxwjaNgUuBNma2GUDS/cB04MEDWTDn3MGnNN81XpUvXVq4zTnn9kqiNo2LmnThSYI+wR+A\nOZLGh59Pp5Dn9ZxzrjACEvReSZE1wtw7w3OAdyO2f3XgiuOcO2gp+hmoY62oSReej2VBnHMHv0Rt\nGkdz17iJpKGSZklakPuKReEORhM/nEDnjkfRqX1Lnnvq0T32b9u2jd8PvIRO7VvS67ROrFi+FIDt\n27fzp+uu5LTftuOMEzvw5eef7Dpm1NvDOb1Te049oQ0P3H17rC5ln0wY/x5Ht2pGq+ZNefSRh/bY\nv23bNi65+AJaNW9KpxOOZdnSpbv2Pfrwg7Rq3pSjWzXj/Qnjo84zUZzWsQkzX/493wy5lpsuPmGP\n/Q3qVmPs45cw+fmrGP9UPzLqVNm1r+8ZRzP71d8z+9Xf0/eMo3dtv3vgyXz3xvWsHXdLTK5hf+Q2\njYt7xUM0YwJfBP5LcB3dgDeAYQewTAetnJwc7vjzDbz0xkg+/GIGo0a8wYJv5+VJM+zVF6lWvTqf\nTZ3LFdcM4sG/3QHA6y+/AMD7n3/NkLfe5d47b2Xnzp1s+GE9D9x1G6+/PY4Pv5jO2jWr+fyTj2J+\nbdHIycnhxuuvZeTocUyfNZfhQ19n3ty5edK8+MLz1KhegznfLmTQDX/g9r8E/8DnzZ3L8GFDmTZz\nDqPGvMcNg35PTk5OVHkmgpQU8dQNXel9y2u0ueyf9DnlSJofVjtPmgev6cKQCbPoOHAwD7z0Gfdc\neQoANaqU5/bLTuTEa16g09UvcPtlJ1K9cnkAxn65gE5XvxDz69lXKVKxr7iUK4o0Fc1sPICZLTKz\nOwgCottLM6ZNoWGjJhzWsDFly5al59l9mDBudJ40E8aN5rwLLwHgzF7nMOnTjzEzvps/jxM6dQag\ndp1DqFq1GrOmf83ypUto2LgptWrXAeC3J53CuNHvxPS6ojVl8mSaNGlKo8bB9fe54ELGjB6ZJ82Y\n0SPp2+8yAM459zwmfvQhZsaY0SPpc8GFlCtXjoaNGtGkSVOmTJ4cVZ6JoEPzdBZlbWDpqh/5dcdO\nhn80hx6/aZYnTfPD6vDJtKUAfDJ96a79p3VowodTF7Ph5638uGkrH05dzOkdmwAweW4Wq3/YFNNr\n2VdS6Q6E28JJFxZJulpST6BKcQe5Pa1elU16Ruauz/XSM/h+VfaeadKDNGlpaVSpWpUNP6ynxZFH\n8f5777Jjxw6WL1vCNzOnk521ksMaN2Hxwu9YsXwpO3bsYMLY0WRnr4zpdUUrOzuLzMzd8/1mZGSS\nlZW1Z5r6QZq0tDSqVqvG+vXrycra89js7Kyo8kwE6XWqsnLt7tUtstb+lKfpCzB70ff0PrE5AL07\nNadqpXLUrFqB9DpV8h37M+l1Suc/wdI8oPoPQCXgeuB+oBowoLiDJOUAswnmMtwBvAw8aWY7JbUH\nLjWz6/e14Mnmgr79WbhgPj1OPYGMzAa063gcqampVK9eg/sfe4ZrB/YjJSWFdh2PY9mSQifidQns\ntn++z5M3dOOSrscwaeYystb+RM7Og2uip1J31ziXmf0vfPsz0G8v8t5iZq0BwjVFXwOqAneZ2VRg\n6l6WNWqS0sJZbBPKofXSyc7aXVtblZ1F3Xrpe6bJXkm9jEx27NjBzz/9RI2atZDEXffvvrlydtfO\nNGpyOACnde3OaV27AzDkpf+Qkpoag6vZe+npGaxcuXvxsayslWRkZOyZZsUKMjOD6/9p40Zq1apF\nRsaex6anB8cWl2ciyF77E5l1dq+HllGnKllrf86TZtX6TVx453AAKlUow1kntWDjpm1kr/2ZTq0P\nizi2Cp/NWBabgpcgEb+mb3EKbRpLelvSiMJee3OScE3Rq4DrFOgsaUx4npMkzQhf0yVVkVRZ0oeS\npkmaLWnXmqWS/hqucP+5pNcl3RRunyjpKUlTgRsk1ZH0lqQp4es3YbpKkl6QNDk8X/71UA+YY9q0\nZ8nihSxftoTt27cz+u3hnNatR540p3XtwZtDXwVg7KgRnNCpM5LYsnkzm3/5BYBPP/6A1LRUjmje\nAoB1a4MlW3/8cQOvvDCYiy65PFaXtFfad+jAwoXfsXRJcP3Dhw2le49eedJ079GLIa+8BMCIt97k\npJNPQRLde/Ri+LChbNu2jaVLlrBw4Xd06NgxqjwTwdT52TTNrMlhh1anTFoKfU5pxbtf5B18Uata\nhV1Nw5sv/i0vjZ0BwPtTFtGlQ2OqVy5P9crl6dKhMe9PWRTrS9h/ItrlPGOuqBrhsyV5IjNbHK5Q\nn3/F+ZuAa81skqTKwNZw+9lm9lO48MpXkkYB7YFzgWMImtzTgK8j8iqbuwygpNcImuKfS2pAsIZB\nC+B24CMzGyCpOjBZ0gdm9ktkocK1VK8CyMgsmXWs0tLSuPfhp+jXpyc5OTlccPFlNGvekscf/BtH\ntW7H6d16cMEl/bnxmgF0at+S6tVr8ux/XgZg3bo19DuvJykpKdStl85T/9x9p/Duv/yJud/MBuDG\nm/9C46aHl0h5S1paWhpPPv0sPbufQU5ODpf1H0DLVq245+47aduuPT169qL/gIEM6N+PVs2bUqNG\nTV4ZMhSAlq1acW6f82lzdEvS0tJ46pnnSA1rvgXlmWhycow/PP0eox+9mNQU8dK4mcxbupa/Xn4S\n0+av4t0vFnBi64bcc+XJmMHns5Zz41PjANjw81YefPkzPv/3QAAeeOkzNvwc/DO5/3enckGXI6lY\nrgwLh9/Af9+dzv0vfhq36yxOok5dpWBq/wOQsbTJzCrn2/Yj0IwgIN1kZj0k3QqcDQwBRpjZSkll\ngCcJ1knZGR7TiGC1qhpmdleY3xNAtpk9JmkiQbP7k3DfGiDyTkSdMJ+JQHmCfkuAmsAZZpZ3HEuE\no1u3s3c/+mKfv4vSrk7VcvEuQtzU6HJvvIsQV1s/ufPrElhjGIC6TY+0Cx57s9h0fz+7RYmdM1rR\nzke43yQ1BnKANQSBEAAze0jSu8CZwCRJZwDHEQSudmb2q6SlBMGrOJG1uhTgODPbGplAwdD2c81s\n/v5cj3Nu76UlaJUwJsWSVIdg5btnLV8VVFITM5ttZg8TTObQnODO9JowCJ4M5PYUTwJ6SiofNqPz\ndrDlNQEYFHGe1uHb8cCgMCAiqc3+X6FzrjjB8Jj9X87zQIi6RiipnJlt24u8K0iawe7hM68ATxSQ\n7sYw2O0kmOBhHME4xdGSZhPcXf4WwMymhH2Fs4DvCYbnbCzk/NcDz0maRXCdnwJXA/cCTwGzwvGR\nSyg6oDrnSkiCjp6Jal3jjsDzBLW0BpKOAa4ws0FFHWdmhY7hMLOJBH11FJLPNuD4Qg5/zMzullSR\nILh9HebTOd851gEXFHDuLcDviiq7c67kJfKaJdE0jZ8hqDGtBzCzmcDJB7JQxRgc1jSnAW+Z2bQ4\nlsU5txdSonjFQzRN4xQzW5av7Z5zgMpTLDO7OF7nds7tnwQdTx1VIFwRNo8tHAc4CPBpuJxze0VS\nwjaNowmE1xA0jxsQ3KD4INzmnHN7JUHjYFTPGq8hGMjsnHP7LJiYNTEjYTR3jf+PYNGmPMzsqgNS\nIufcwUmQWooHVH8AfBi+JhE8K7w34wmdcw4IZqAp7r+o8pG6hpOvLAwf0y0s3bmSLJz6r1DRNI3z\nTMsv6RXg86hK65xzoZJazjO8afsccBqwEpgiaZSZzc2XrgpwA/C/PXPJa18qqo2AuvtwnHMuyaWm\nqNhXFDoCC81ssZltB4YCBU2ndy/wMLtntCpUNKvYbZD0Q/j6EXgfuC2a0jrnXK4SXMUuA1gR8Xll\nuG33uaS2QH0zi1yTvVBFNo3DiQmOAXIXgdiZf9IE55yLSvRrktQOJ1jONdjMBkd9mmAOgSeA/tEe\nU2QgNDOTNNbMjow2Q+ecK4iAtOiqfOuKmY8wC4icLTmT3ZU1CCZtORKYGD4RdygwSlKvcJmQPUTT\nRzjDp6pyzpWEElrFbgpwuKRGksoSjHMelbvTzDaaWW0za2hmDYGvgEKDIBRRI9TuBZDaENyVWUQw\n8amCc1li0T8lAAAZnElEQVTbqIrsnHMAiJQoh8cUxcx2SLqOYG7RVOAFM5sj6R5gqpmNKjqHPRXV\nNJ4MtAUSbyUc51ypoxIcUG1mY4Gx+bbdWUjazsXlV1QgVJhJKVwuyzmXiErjI3Z1JP2xsJ1mVtBs\n0845VyBROqfhSgUqQwk06p1zjsSdobqoQLjKzO6JWUmccwc1kbjrGhfbR+iccyUiXMUuERUVCE+N\nWSmccwc9AamlLRCa2Q+xLIhz7uCXmGFwL9Y1ds65/ZWgFUIPhM652BAqfU1j55wraaXxZokLpaaI\nKuX9q0pKG7+PdwkOKokZBj0QOudiRCqFd42dc66kedPYOZf0EjMMeiB0zsVIqRxQ7ZxzJS1B46AH\nQudcrKhUzkfonHMlJph9xgOhcy6ZRb84U8x5IHTOxYw3jZ1zSU1Agk5Q7YHQORc78j5C51yy86ax\ncy6pedPYOeeQN42dc0lOXiN0ziW5oGmcmJHQA6FzLmYSMwx6IHTOxZDPR+icS3oJGgc9EDrnYidB\n4yAp8S6Acy45iKBpXNwrqrykrpLmS1oo6dYC9v9R0lxJsyR9KOmwovLzQOici41w9pniXsVmI6UC\nzwHdgJbARZJa5ks2HWhvZkcDbwKPFJWnB0LnXMwoilcUOgILzWyxmW0HhgK9IxOY2cdmtjn8+BWQ\nWVSG3kfonIuRqJu+tSVNjfg82MwGR3zOAFZEfF4JHFtEfgOBcUWd0GuEMfbBhPfo0LolbY9qxpOP\nPbzH/m3btjHg0otoe1Qzupx0PMuXLc2zf8WK5WQeUo2/P/V41Hkmkgnj3+PoVs1o1bwpjz7y0B77\nt23bxiUXX0Cr5k3pdMKxLFu6dNe+Rx9+kFbNm3J0q2a8P2F81HkmitNOaMHMt//KNyPv4qbLT9tj\nf4N6NRj7r0FMHnYb4//vBjIOqb5rX/1DazD6H9cy/a07mPbW7TSoVxOAw9Jr8enLN/HNyLt45aHL\nKZOWGrPr2RdRNo3XmVn7iNfgYrIt4ny6BGgPPFpUOg+EMZSTk8PNf7ye4W+P4auvZ/PW8GF8O29u\nnjSvvPQC1arXYNrs+Vxz3Y3c/dfb8uy/49ab6HJ6173KM1Hk5ORw4/XXMnL0OKbPmsvwoa8zb27e\nsr74wvPUqF6DOd8uZNANf+D2v9wCwLy5cxk+bCjTZs5h1Jj3uGHQ78nJyYkqz0SQkiKeuvV8el/3\nD9qcex99urajeeND86R58A9nM+TdyXS84EEeGDyOewb12rXvP/deypMvfUibc++j0yWPsnbDzwDc\nf0Nv/j7kY47s/Tc2/LyF/mcfH9Pr2hvRNIujbBpnAfUjPmeG2/KeT+oC3A70MrNtRWXogTCGvp46\nmcaNm9CwUWPKli3LOeedz9gxo/KkGTdmFBf17QdA77PP5ZOJH2FmALw7eiQNDmtI8xYt9yrPRDFl\n8mSaNGlKo8ZBWftccCFjRo/Mk2bM6JH07XcZAOecex4TP/oQM2PM6JH0ueBCypUrR8NGjWjSpClT\nJk+OKs9E0OHIhixasY6lWev5dUcOw8dPo0fno/Okad64Hp9Mng/AJ1MW0KPzUeH2Q0lLTeGj/30L\nwC9btrNl668AnNThCEZ8MB2AIaP/R8/Ox8TqkvZJCd01ngIcLqmRpLLAhUCeX3pJbYB/EwTBNcVl\n6IEwhlZlZ5ORufsPWXpGJqtWZedJkx2RJi0tjapVq/HD+vVs2rSJp594hFv+cude55kosrOzyIwo\na0ZGJllZWXumqR9x/dWqsX79erKy9jw2OzsrqjwTQfoh1Vj5/YZdn7O+30BGnWp50sxekEXvU1oD\n0PuUY6hauQI1q1Xi8AaH8OPPWxj62BV8+fotPHDjWaSkiFrVK7Hx5y3k5OzclWf6IXnzTDQlcdfY\nzHYA1wHjgXnAG2Y2R9I9knKr0Y8ClYHhkmZIKrJ2EJebJZJygNlAGWAH8DLwpJntlNQeuNTMro9H\n2RLVw/f/jWuuu5HKlSvHuyjuALntybd58pY+XNLrWCZNW0jW9xvIydlJWloKv2nThOMueogVqzfw\n6sMD6NfrOMZMnBXvIu+1khpQbWZjgbH5tt0Z8b7L3uQXr7vGW8ysNYCkQ4DXgKrAXWY2FZha1MGl\nVb30dLJW7r7ZlZ21knr10vOkSQ/TZGRksmPHDn76aSM1a9Vi6tTJjHxnBHfdcSsbN/5ISkoK5cqX\np3WbtsXmmSjS0zNYGVHWrKyVZGRk7JlmxQoyM8Pr37iRWrVqkZGx57Hp6cGxxeWZCLLXbCSzbo1d\nnzPq1iBr7cY8aVat3ciFN/0HgEoVynLWqa3ZuGkLWd//yKwFK1matR6AUR/PpONRjXjpnS+pVqUC\nqakp5OTsJKNuDbLX5M0zoShxnzWOe9M4bL9fBVynQGdJYwAknRRWa2dImi6pSrj9FkmzJc2U9FC4\nbWJYm0RSbUlLw/etJE0O85gl6XBJlSS9Gx7/jaQLYnGtbdt1YNGihSxbuoTt27cz4s036Na9Z540\nXbv35PUhrwAw8u23OPGkk5HEuPc/Yda8Rcyat4hrrr2eP950K1ddfW1UeSaK9h06sHDhdyxdEpR1\n+LChdO/RK0+a7j16MeSVlwAY8dabnHTyKUiie49eDB82lG3btrF0yRIWLvyODh07RpVnIpg6ZxlN\nG9ThsPRalElLpc8ZbXk3X42uVvVKuwLFzQPO4KWRX+06tlqVCtSuEbQGOndoxreLVwPw6dQFnNOl\nDQB9ex6b0LXE4MmS/W8aHwgJMY7QzBaHo8UPybfrJuBaM5skqTKwVVI3gsGTx5rZZkk1i8n+auBp\nMxsSdqymAmcC2WbWHUDSHh0rkq4iCNBk1m+wP5e3S1paGo88/jTn9j6TnJwc+l7anxYtW/HAvXfR\num17zuzek36XDeDqKy6j7VHNqFGjBs+/9No+5ZmI0tLSePLpZ+nZ/QxycnK4rP8AWrZqxT1330nb\ndu3p0bMX/QcMZED/frRq3pQaNWryypChALRs1Ypz+5xPm6NbkpaWxlPPPEdqajBUpKA8E01Ozk7+\n8PAbjP7HtaSmiJdGfsW8xav56zXdmTZ3Oe9+MpsT2x/OPYN6YQafT1vIjQ++AcDOncZtT7zD2H8N\nQhLT5y3nhRGTALj96ZG88tDl3PX7Hsycv4IX3/kynpdZrMSsD4Jy70jG9KTSJjOrnG/bj0AzoAVw\nk5n1CJ8hPBsYAowws5WSHge+NbP/y3f8xPC4qZJqA1PNrKGkiwluob8c5vGdpCOACcAwYIyZfVZU\nedu0bW8ff/6/krj0Uql82cQem3Yg1ehwXbyLEFdbZzz3tZm1L4m8jjymrb353ufFpmuRXqnEzhmt\nuDeNASQ1BnKAPLe5zewh4AqgAjBJUvMistnB7uspH5HHa0AvYAswVtIpZrYAaEtww+Y+SXfmz8w5\nV/IStWkc90AoqQ7wL+BZy1c9ldTEzGab2cMEY4eaA+8Dl0uqGKbJbRovBdqF78+LyKMxsNjMngFG\nAkdLSgc2m9mrBLfZ2x6o63PO7VZCA6pLXLz6CCtImsHu4TOvAE8UkO5GSScDO4E5wDgz2yapNTBV\n0naCW+h/AR4D3gj79t6NyON8oJ+kX4HVwANAB+BRSTuBX4FrDsRFOud2y52GKxHFJRCaWaGdTmY2\nEZgYvh9USJqHgIfybfsWiByqf0dhaQkGYo7HORc7cWz6Fich7ho755JDgsZBD4TOuViJfgbqWPNA\n6JyLmQSNgx4InXOxkftkSSLyQOicixklaC+hB0LnXMx4jdA5l9wEKR4InXMuMSOhB0LnXEz4zRLn\nnMObxs4553eNnXMuQeOgB0LnXGzI7xo755w3jZ1zzpvGzjnnTWPnXJKTN42dc8nNB1Q75xweCJ1z\nzpvGzrkk54s3OeeSnfcROuccids0Tol3AZxzyUMq/hVdPuoqab6khZJuLWB/OUnDwv3/k9SwqPw8\nEDrnYqYkAqGkVOA5oBvQErhIUst8yQYCG8ysKfAk8HBReXogdM7FjKL4LwodgYVmttjMtgNDgd75\n0vQGXgrfvwmcqiIWVfY+wijMmP71uhqV0pbFsQi1gXVxPH88JfO1Q/yv/7CSymj6tK/HVyyr2lEk\nLS9pasTnwWY2OOJzBrAi4vNK4Nh8eexKY2Y7JG0EalHId+mBMApmViee55c01czax7MM8ZLM1w4H\n1/WbWdd4l6Ew3jR2zpU2WUD9iM+Z4bYC00hKA6oB6wvL0AOhc660mQIcLqmRpLLAhcCofGlGAZeF\n788DPjIzKyxDbxqXDoOLT3LQSuZrB7/+PYR9ftcB44FU4AUzmyPpHmCqmY0CngdekbQQ+IEgWBZK\nRQRJ55xLCt40ds4lPQ+Ezrmk54HQOZf0PBA655KeB8KDgCT/OSahoh4Zc3vHh8+UMpJkZibpGIKh\nAxvMbEnu9niXL14ivpdqQIqZbYh3mQ6kyJ+3pG7AWjObWsxhrhBekyhlwn/sXYHXgS7APElHJXMQ\nhF3fS2/gDeA1SbdKKh/vch0oEUHwBuB+grFybh95ICxFFMgE/gz0BGYAi4HvI9PEqXhxJakjcDNw\nMTCN4GmC1LgW6gCTdDxwKfBbM1ss6QRJ5yTr78D+8AHVpYykcsD1wM/A5cAlZvadpPOAz8zs+yIz\nOEhJOhloAPwKDAIuDrsMmpjZoviWrmTk7/6QVBe4ByhHUCM8EtgETDCzf8WnlKWT9xEmuIi+r+rA\nVmA78NvwdaiZ/SqpPXALsICI2uHBrIA+0U3AAIJWzvlmtkJSD+BGSecT9KWW2r/6+foErwe2ACOB\nt4GzCObemwdcB5SNVzlLKw+ECS4Mgt0Jmn3zCAJdX2A28LCkzUAP4C4zmxW/ksZOxB+H7kA7oCJw\nJ8HD+I2BhmFT+W/ArWZW6vvPIoLgdQTN/75mtgZ4L3wh6TKC341+8SpnaeVN4wQn6TjgHwS//L2B\ny8ysZdgsOotgcbA5ZvZZMt05lnQ68ABBLfAdYKSZ/UHSfUBVgglNXzaz90rz9xIR9FOASsAQggC/\nDOgOHA58CawG7gJuN7PZ8SpvaeWBMAFF/PKXBVoTzKybAvwVuNDMlko63My+i2tB4yicaWQUUAe4\nA+hnZosj9lc0s83xKl9Jk9Q4vCHyFEFfaFlgPlAd+I5gTY7KZvZzHItZankgTFCSzgDaAouApwkm\nmuxkZlskdSa4UXKTma2NXyljT1InoAzwG6ApwZTs14Q3jC4GapvZM5JSzGxnPMtaUsIV2KYB5wOT\ngROAuWa2XNKlwEXA2Wa2NW6FLOV8+EwCknQ0cA7BZJJvAP8mmGE3U9K5wLPA8CQMgi2BR4G5wEcE\nXQXPh0GwI0HN8BuAgyUIApjZUuAK4J9AZzN7D1gtaSDBTbI/eRDcP36zJMFIqkSw6tYaYDmAmd0d\nDg27j+CP181mNq40933tLUlHEgTBSWa2miAQXAXcHg4wbwbcYmYfxbOcJUnSqQS/AwvNbISk7cDf\nJW0P+z4zgfPMbF58S1r6edM4gYS/2KuB5gRLFA42s2ci9qcAaeEShkkjfGxuC8Gsw7WBa4DlZrZT\nUuNwX0UzW1Sa/zgUME7wvwSVlbuAZWaWI+khgiEyZwBflNZrTTTeNI6z3KcAJB1LcHf4djP7huAu\n8SBJ1+SmNbOdyRIEI76XlgQ14RYEi3avA/4EpAOEa9uuyh00XVoDQ75xghdJOtPMLgd+AW4HGoVJ\n5xD8QVhTWq81EXmNMAFIOhO4FfgWOAn4h5k9HfYVjgfuN7Nn41nGeAjHCd5IEAQmA48T9AH+G9hB\nMHYy/+plpVr47HBfYED4BxFJ/yIYOvMrcBxwupmtjF8pDz4eCONMUi3gVeBRM/soHB93BTDDzB6Q\n1AaoZmYT41nOWAubvG8D54ab+hEMFfk/YCHwX+A+M5sTnxKWPEn1CK5voJl9L6mcmW0L93UjWGz9\nE+8TLHl+syTGJDUD2hB0+q8ws/WSVgAtJH1iZhPCwdIPSMo2sxfD40pt39c+qkTQ97fSzLaG/WWD\nCZ4guZPgWeJS/X0U8DPdSPDccBvgvYgg2MrMxsWjjMnC+whjKOz3ugp4BXhE0tOSKhPMIlOZ4Plh\ngK8Jaj03SWoOpbfvK1oRfYKpAOHTETOBAZJqhkNIRhDMKNOntH8f+foEr5XUi6Dp+zHQTNJR4b6L\ngLsk1YhfaQ9+3jSOMUmnEdwFvAx4gqDP6xCCR+V+IXhSog3B41PXAuPM7IP4lDY2Ip6k6QmcCFQg\nuCFyGtAZqAdMAG4C7gWuJJh1p9RPMBHRJ9jfzOZKak0wWL4xQQ2xHcEQmYOmCyAReY0wxszsfYI7\nnxebWW+Cmt+ZwPEENwB+ALoRPEbVM9x/UAuD4JkEU0r9FziZoH9wFkGf2SygI8Efj2VAeWBbfEq7\nfyS1D8dEIukQgj945wJZkvoQTKU1DLiNYAjVGR4EDzyvEcZQRM2nA9CLYOD0EOApYAnBY2NvE0y1\n9QbBDCPfxKu8sRA2idOAvxP0AaYTTDy7GmhI0AxeFo6hPIMgWF5pZjPiU+J9Fz47fjbwCbDNzDZI\neovgj191ghrg4cAbZvZc/EqafDwQxkFYE3iFoBl4o5n9O9y+a6IASXUPhqZfYSL+KKSbWbakCgRd\nBK8D3cMgsRKYThAMt0o6EVhV2iebkHQEQU33CoLJE44luHk2X9KVQKdw36+lvS+0tPCmcRxYMI/c\nHQQ3SUZB8NSImW0Oaz4c5EEwJQyC3YEJkg4zsy0Ek6tmEdwsaEcwhvKh3OdozezT0hgEJR0u6XhJ\np0iqbmYLgDEEjwyWMbMXgAXhs8M3EFzzdg+CseOBMH5mEDwl0ClyppSDabKA/BQuphQ+Gvdb4DGC\nMXPLFMzAbQQTzl5LMPvycDObFLcCl4Aw2A8jaO7fBswKB8r/HZgI3Bt2lVQk6Cc+38zmxqm4Scub\nxnEU/gMoY2ZfxLssB1o4/OMu4K9m9rOks4FMglml2wBXA28BYwlm4a5VGvsBI4WTQdxNMBnEJ+G2\nuwgmk+1uZt8omHb/PIKm8BIz+zVe5U1mHghdTEiqSTBIOpVg4oTNBI/MVSB4smYFwbCRZ83s83iV\ns6SE17sO6GVmYySVz23iS7qb4EmZYwj6CPsAY81sRbzKm+w8ELoDKrfZH/H/PxGssTKQoD+wgpn9\nGD5S9ybBHeGv41nmkhI2ix8imENwfb5H5j4mmEdwmqRUM8uJa2GTnD9i5w6Y8O7opeE0WpL0IMFs\n27+G/7/PzP6nYGH2u4G/HSxBEMDM3pW0E5gsqX14J7xM2Pz9ieB7wINg/PnNEndAhM9UjwDWs3tQ\n+FSCNVj+STDD9K2S2hI8Uni1mb2T+6jdwSJ8Rvg6YKqkGhYsv3opcChJsvRqaeA1QlfiwjkEhwB/\nMbNREdtXEQwXOo5gEHkacD9wbu60UgfjkBELZhO/DvhU0j8I+gcHhsOoXALwPkJX4sKhMZ+aWUr4\nuUI4ThBJTxAMnL6U4KZJxXBChYOeggXnRwBt/LG5xOJNY1fiwru+3SUtklTLgpX3yoe7/0fwB3in\nma1JliAIYGZjgOoeBBOPB0J3QET0jU0Op9HKXWVtG/CjpLK5T9EkEzuI1lo+mCTdL6KLncgbBbDr\nBspDwOjwEbKD9ikaV7p4H6E74MJp5t8imGHnZjMbG+ciOZeHB0IXEwrW6K1qZm/HuyzO5eeB0MVU\nEq694koBD4TOuaTnN0ucc0nPA6FzLul5IHTOJT0PhC4PSTmSZkj6RtJwSRX3I6/OksaE73tJurWI\ntNUl/X4fznG3pJui3Z4vzYuSztuLczWUdFAvppWsPBC6/LaYWWszO5JgNb2rI3cqsNe/N2Y2yswe\nKiJJdWCvA6FzJcEDoSvKZ0DTsCY0X9LLBAvS15d0uqQvJU0La46VIZieXtK3kqYB5+RmJKm/pGfD\n93UlvS1pZvg6geCJkyZhbfTRMN3NkqZImiXpbxF53S5pgaTPgWbFXYSkK8N8Zkp6K18tt4ukqWF+\nPcL0qZIejTj37/b3i3SJzQOhK5CkNIKF5meHmw4H/mFmrYBfCFbh62JmbQkeoftjOLHC/xEsTN+O\nYM69gjwDfGJmxwBtCRaxuhVYFNZGb5Z0enjOjgRzGLaTdKKC1e0uDLedCXSI4nJGmFmH8HzzCGbH\nztUwPEd34F/hNQwENppZhzD/KyU1iuI8rpTy+QhdfhUk5S6a9BnwPMGi68vM7Ktw+3FAS2BSOI9q\nWeBLoDnBAkTfAUh6FbiqgHOcQjANV+7szBvDxZ0inR6+poefKxMExirA2xHrP4+ieEdKuo+g+V2Z\nYJnQXG+Ezzx/J2lxeA2nA0dH9B9WC8+9IIpzuVLIA6HLb4uZtY7cEAa7XyI3Ae+b2UX50uU5bj8J\neNDM/p3vHDfuQ14vAmeZ2UxJ/YHOEfvyP1Fg4bkHmVlkwERSw304tysFvGns9sVXwG8kNQWQVClc\nn+RboKGkJmG6iwo5/kPgmvDY1HBNk58Janu5xgMDIvoeMyQdAnwKnCWpgqQqBM3w4lQBVkkqA/TN\nt6+PpJSwzI2B+eG5rwnTI+kISZWiOI8rpbxG6Paama0Na1avSyoXbr7DzBZIugp4V9JmgqZ1lQKy\nuAEYLGkgkANcY2ZfSpoUDk8ZF/YTtgC+DGukm4BLwlXfhgEzgTUE6yIX568EE8KuDf8fWablwGSg\nKsG6KVsl/Yeg73CagpOvBc6K7ttxpZE/a+ycS3reNHbOJT0PhM65pOeB0DmX9DwQOueSngdC51zS\n80DonEt6Hgidc0nv/wFLuednwfOSmwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x162cd74a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "matrix = score_model(predicted_l)\n",
    "utils.plot_confusion_matrix(matrix, classes=[\"Agree\",\"Disagree\", \"Discuss\"],\n",
    "                      title='Normalized confusion matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0, 155, 17], [0, 1723, 71], [0, 553, 165]]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "true_label = [(2 if x[-1] == \"agree\" else (1 if x[-1] == \"discuss\" else 0)) for x in stances_val.values]\n",
    "[list(x) for x in list(confusion_matrix(true_label,predicted_l))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), './saved_models/CNN_refuting_ft.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train finalized model below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_final = CNN(EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fts = train_feats+val_feats\n",
    "tr_labels = [str(x[-1]) for x in stances_tr.values]+[str(x[-1]) for x in stances_val.values]\n",
    "\n",
    "def train_final(model, total_batch, optimizer, criterion):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    model_final.train()\n",
    "    for i in range(total_batch):\n",
    "        batch_x,batch_y = get_batch(fts, tr_labels,i,batch_size)\n",
    "        inputs = Variable(torch.FloatTensor(batch_x))\n",
    "        labels = Variable(torch.LongTensor(batch_y))\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        predictions = model_final(inputs)\n",
    "        loss = criterion(predictions, labels)\n",
    "        _, predicted = torch.max(predictions.data, 1)\n",
    "        acc = binary_accuracy(predicted, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "    return epoch_loss / total_batch, epoch_acc / total_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Epoch: 01 | Train Loss: 0.727 | Train Acc: 69.69% |\n",
      "| Epoch: 02 | Train Loss: 0.555 | Train Acc: 77.40% |\n",
      "| Epoch: 03 | Train Loss: 0.465 | Train Acc: 80.90% |\n"
     ]
    }
   ],
   "source": [
    "batches_train= int(len(fts)/batch_size)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    train_loss, train_acc = train_final(model_f, batches_train, optimizer, criterion)\n",
    "    print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}% |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.save(model_final.state_dict(), './CNN_model_softmax_final.pth')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}