CNN_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "from collections import Counter\n",
    "from tqdm import tqdm_notebook\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1. Dataset preparation\n",
    "\n",
    "\n",
    "#### Step 1.1 Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_data = pd.read_csv('training.txt', sep='\\t')\n",
    "sentiment_data.columns =['Class', 'Data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>this was the first clive cussler i've ever rea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>i liked the Da Vinci Code a lot.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>i liked the Da Vinci Code a lot.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>I liked the Da Vinci Code but it ultimatly did...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>that's not even an exaggeration ) and at midni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class                                               Data\n",
       "0      1  this was the first clive cussler i've ever rea...\n",
       "1      1                   i liked the Da Vinci Code a lot.\n",
       "2      1                   i liked the Da Vinci Code a lot.\n",
       "3      1  I liked the Da Vinci Code but it ultimatly did...\n",
       "4      1  that's not even an exaggeration ) and at midni..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1.2 Shuffling data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "sentiment_data = shuffle(sentiment_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1.3 Creating the Vocab and the vocab2int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = sentiment_data.iloc[:, 0].values\n",
    "reviews = sentiment_data.iloc[:, 1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_processed = []\n",
    "unlabeled_processed = [] \n",
    "for review in reviews:\n",
    "    review_cool_one = ''.join([char for char in review if char not in punctuation])\n",
    "    reviews_processed.append(review_cool_one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_reviews = []\n",
    "all_words = []\n",
    "for review in reviews_processed:\n",
    "    word_reviews.append(review.lower().split())\n",
    "    for word in review.split():\n",
    "        all_words.append(word.lower())\n",
    "    \n",
    "counter = Counter(all_words)\n",
    "vocab = sorted(counter, key=counter.get, reverse=True)\n",
    "vocab_to_int = {word: i for i, word in enumerate(vocab, 1)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1.4 Encoding words to ints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_to_ints = []\n",
    "for review in word_reviews:\n",
    "    reviews_to_ints.append([vocab_to_int[word] for word in review])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1.5 Checking if there was any review with length == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zero-length 0\n",
      "Max review length 931\n"
     ]
    }
   ],
   "source": [
    "reviews_lens = Counter([len(x) for x in reviews_to_ints])\n",
    "print('Zero-length {}'.format(reviews_lens[0]))\n",
    "print(\"Max review length {}\".format(max(reviews_lens)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1.6 Padding the data to the same sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_len = 250\n",
    "\n",
    "features = np.zeros((len(reviews_to_ints), seq_len), dtype=int)\n",
    "for i, review in enumerate(reviews_to_ints):\n",
    "    features[i, -len(review):] = np.array(review)[:seq_len]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1.7 Creating training and testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_trian shape (6400, 250)\n"
     ]
    }
   ],
   "source": [
    "X_train = features[:6400]\n",
    "y_train = labels[:6400]\n",
    "\n",
    "X_test = features[6400:]\n",
    "y_test = labels[6400:]\n",
    "\n",
    "print('X_trian shape {}'.format(X_train.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 Define a model\n",
    "\n",
    "\n",
    "#### Step 2.1 Define functions for creating weights and biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weights_init(shape):\n",
    "    return tf.Variable(tf.truncated_normal(shape=shape, stddev=0.05))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bias_init(shape):\n",
    "    return tf.Variable(tf.zeros(shape=shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 2.2 Define helper functions for the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_inputs(batch_size, sequence_len):\n",
    "    '''\n",
    "    This function is used to define all placeholders used in the network.\n",
    "    \n",
    "    Input(s): batch_size - number of samples that we are feeding to the network per step\n",
    "              sequence_len - number of timesteps in the RNN loop\n",
    "              \n",
    "    Output(s): inputs - the placeholder for reviews\n",
    "               targets - the placeholder for classes (sentiments)\n",
    "               keep_probs - the placeholder used to enter value for dropout in the model    \n",
    "    '''\n",
    "    inputs = tf.placeholder(tf.int32, [batch_size, sequence_len], name='inputs_reviews')\n",
    "    targets = tf.placeholder(tf.float32, [batch_size, 1], name='target_sentiment')\n",
    "    keep_probs = tf.placeholder(tf.float32, name='keep_probs')\n",
    "    \n",
    "    return inputs, targets, keep_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embeding_layer(vocab_size, embeding_size, inputs):\n",
    "    '''\n",
    "    Function used for creating word embedings (word vectors)\n",
    "    \n",
    "    Input(s): vocab_size - number of words in the vocab\n",
    "              embeding_size - length of a vector used to represent a single word from vocab\n",
    "              inputs - inputs placeholder\n",
    "    \n",
    "    Output(s): embed_expended -  word embedings expended to be 4D tensor so we can perform Convolution operation on it\n",
    "    '''\n",
    "    word_embedings = tf.Variable(tf.random_uniform([vocab_size, embeding_size]))\n",
    "    embed = tf.nn.embedding_lookup(word_embedings, inputs)\n",
    "    embed_expended = tf.expand_dims(embed, -1) #expend dims to 4d for conv layer\n",
    "    return embed_expended"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_conv(input, filter_size, number_of_channels, number_of_filters, strides=(1, 1), activation=tf.nn.relu, max_pool=True):\n",
    "    '''\n",
    "    This is classical CNN layer used to convolve over embedings tensor and gether useful information from it.\n",
    "    \n",
    "    Input(s): input - word_embedings\n",
    "              filter_size - size of width and height of the Conv kernel\n",
    "              number_of_channels - in this case it is always 1\n",
    "              number_of_filters - how many representation of the input review are we going to output from this layer \n",
    "              strides - how many pixels does kernel move to the side and up/down\n",
    "              activation - a activation function\n",
    "              max_pool - boolean value which will trigger a max_pool operation on the output tensor\n",
    "    \n",
    "    Output(s): text_conv layer\n",
    "    \n",
    "    '''\n",
    "    weights = weights_init([filter_size, filter_size, number_of_channels, number_of_filters])\n",
    "    bias = bias_init([number_of_filters])\n",
    "    \n",
    "    layer = tf.nn.conv2d(input, filter=weights, strides=[1, strides[0], strides[1], 1], padding='SAME')\n",
    "    \n",
    "    if activation != None:\n",
    "        layer = activation(layer)\n",
    "    \n",
    "    if max_pool:\n",
    "        layer = tf.nn.max_pool(layer, ksize=[1, 2, 2 ,1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "    \n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_layer(lstm_size, number_of_layers, batch_size, dropout_rate):\n",
    "    '''\n",
    "    This method is used to create LSTM layer/s for PixelRNN\n",
    "    \n",
    "    Input(s): lstm_cell_unitis - used to define the number of units in a LSTM layer\n",
    "              number_of_layers - used to define how many of LSTM layers do we want in the network\n",
    "              batch_size - in this method this information is used to build starting state for the network\n",
    "              dropout_rate - used to define how many cells in a layer do we want to 'turn off'\n",
    "              \n",
    "    Output(s): cell - lstm layer\n",
    "               init_state - zero vectors used as a starting state for the network\n",
    "    '''\n",
    "    def cell(size, dropout_rate=None):\n",
    "        layer = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "        \n",
    "        return tf.contrib.rnn.DropoutWrapper(layer, output_keep_prob=dropout_rate)\n",
    "            \n",
    "    cell = tf.contrib.rnn.MultiRNNCell([cell(lstm_size, dropout_rate) for _ in range(number_of_layers)])\n",
    "    \n",
    "    init_state = cell.zero_state(batch_size, tf.float32)\n",
    "    return cell, init_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten(layer, batch_size, seq_len):\n",
    "    '''\n",
    "    Used to transform/reshape 4d conv output to 2d matrix\n",
    "    \n",
    "    Input(s): Layer - text_cnn layer\n",
    "              batch_size - how many samples do we feed at once\n",
    "              seq_len - number of time steps\n",
    "              \n",
    "    Output(s): reshaped_layer - the layer with new shape\n",
    "               number_of_elements - this param is used as a in_size for next layer\n",
    "    '''\n",
    "    dims = layer.get_shape()\n",
    "    number_of_elements = dims[2:].num_elements()\n",
    "    \n",
    "    reshaped_layer = tf.reshape(layer, [batch_size, int(seq_len/2), number_of_elements])\n",
    "    return reshaped_layer, number_of_elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dense_layer(input, in_size, out_size, dropout=False, activation=tf.nn.relu):\n",
    "    '''\n",
    "    Output layer for the lstm netowrk\n",
    "    \n",
    "    Input(s): lstm_outputs - outputs from the RNN part of the network\n",
    "              input_size - in this case it is RNN size (number of neuros in RNN layer)\n",
    "              output_size - number of neuros for the output layer == number of classes\n",
    "              \n",
    "    Output(s) - logits, \n",
    "    '''\n",
    "    weights = weights_init([in_size, out_size])\n",
    "    bias = bias_init([out_size])\n",
    "\n",
    "    layer = tf.matmul(input, weights) + bias\n",
    "\n",
    "    if activation != None:\n",
    "        layer = activation(layer)\n",
    "\n",
    "    if dropout:\n",
    "        layer = tf.nn.dropout(layer, 0.5)\n",
    "\n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_optimizer(logits, targets, learning_rate, ):\n",
    "    '''\n",
    "    Function used to calculate loss and minimize it\n",
    "    \n",
    "    Input(s): rnn_out - logits from the fully_connected layer\n",
    "              targets - targets used to train network\n",
    "              learning_rate/step_size\n",
    "    \n",
    "    \n",
    "    Output(s): optimizer - optimizer of choice\n",
    "               loss - calculated loss function\n",
    "    '''\n",
    "    loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=targets))\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "    return loss, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SentimentCNN(object):\n",
    "    \n",
    "    def __init__(self, learning_rate=0.001, batch_size=100, seq_len=250, vocab_size=10000, embed_size=300,\n",
    "                conv_filters=32, conv_filter_size=5, number_of_lstm_layers=1, lstm_units=128):\n",
    "        \n",
    "        \n",
    "        '''\n",
    "        To created Sentiment embed network CNN-LSTM create object of this class.\n",
    "        \n",
    "        Input(s): learning_rate/step_size - how fast are we going to find global minima\n",
    "                  batch_size -  the nuber of samples to feed at once\n",
    "                  seq_len - the number of timesteps in unrolled RNN\n",
    "                  vocab_size - the number of nunique words in the vocab\n",
    "                  embed_size - length of word embed vectors\n",
    "                  conv_filters - number of filters in output tensor from CNN layer\n",
    "                  conv_filter_size - height and width of conv kernel\n",
    "                  number_of_lstm_layers - the number of layers used in the LSTM part of the network\n",
    "                  lstm_units - the number of neurons/cells in a LSTM layer\n",
    "        \n",
    "        '''\n",
    "        tf.reset_default_graph()\n",
    "        self.inputs, self.targets, self.keep_probs = define_inputs(batch_size, seq_len)\n",
    "        \n",
    "        embed = embeding_layer(vocab_size, embed_size, self.inputs)\n",
    "        \n",
    "        #Building the network\n",
    "        convolutional_part = text_conv(embed, conv_filter_size, 1, conv_filters)\n",
    "        conv_flatten, num_elements = flatten(convolutional_part, batch_size, seq_len)\n",
    "        \n",
    "        cell, init_state = lstm_layer(lstm_units, number_of_lstm_layers, batch_size, self.keep_probs)\n",
    "        \n",
    "        outputs, states = tf.nn.dynamic_rnn(cell, conv_flatten, initial_state=init_state)\n",
    "        \n",
    "        review_outputs = outputs[:, -1, :]\n",
    "        \n",
    "        logits = dense_layer(review_outputs, lstm_units, 1, activation=None)\n",
    "        \n",
    "        self.loss, self.opt = loss_optimizer(logits, self.targets, learning_rate)\n",
    "        \n",
    "        self.preds = tf.nn.sigmoid(logits)\n",
    "        currect_pred = tf.equal(tf.cast(tf.round(self.preds), tf.int32), tf.cast(self.targets, tf.int32))\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(currect_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 Training and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SentimentCNN(learning_rate=0.001, \n",
    "                     batch_size=50, \n",
    "                     seq_len=250, \n",
    "                     vocab_size=len(vocab_to_int) + 1, \n",
    "                     embed_size=300,\n",
    "                     conv_filters=32, \n",
    "                     conv_filter_size=5, \n",
    "                     number_of_lstm_layers=1, \n",
    "                     lstm_units=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "session.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 5\n",
    "batch_size = 50\n",
    "drop_rate = 0.7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 3.1 Training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02501d1d749c4360a6cf1c50f04b0d32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=128), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/5  | Epoch loss: 0.2494693547487259  | Mean train accuracy: 0.8839062452316284\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2640134157b445b18fee86760509648e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=128), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/5  | Epoch loss: 0.02794887311756611  | Mean train accuracy: 0.9901562333106995\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc5b99a0390f4e1397e9562182f384f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=128), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2/5  | Epoch loss: 0.014880077913403511  | Mean train accuracy: 0.9953124523162842\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8865d60f14741b0b92f90f6c89c760c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=128), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3/5  | Epoch loss: 0.004164558369666338  | Mean train accuracy: 0.9989062547683716\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bba540c6873949419db532993dda966e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=128), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4/5  | Epoch loss: 0.0043505411595106125  | Mean train accuracy: 0.9989062547683716\n"
     ]
    }
   ],
   "source": [
    "for i in range(epochs):\n",
    "    epoch_loss = []\n",
    "    train_accuracy = []\n",
    "    for ii in tqdm_notebook(range(0, len(X_train), batch_size)):\n",
    "        X_batch = X_train[ii:ii+batch_size]\n",
    "        y_batch = y_train[ii:ii+batch_size].reshape(-1, 1)\n",
    "        \n",
    "        c, _, a = session.run([model.loss, model.opt, model.accuracy], feed_dict={model.inputs:X_batch, \n",
    "                                                                                  model.targets:y_batch,\n",
    "                                                                                  model.keep_probs:drop_rate})\n",
    "        \n",
    "        epoch_loss.append(c)\n",
    "        train_accuracy.append(a)\n",
    "        \n",
    "    \n",
    "    print(\"Epoch: {}/{}\".format(i, epochs), \" | Epoch loss: {}\".format(np.mean(epoch_loss)), \n",
    "          \" | Mean train accuracy: {}\".format(np.mean(train_accuracy)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 3.2 Testing process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_accuracy = []\n",
    "\n",
    "ii = 0\n",
    "while ii + batch_size <= len(X_test):\n",
    "    X_batch = X_test[ii:ii+batch_size]\n",
    "    y_batch = y_test[ii:ii+batch_size].reshape(-1, 1)\n",
    "\n",
    "    a = session.run([model.accuracy], feed_dict={model.inputs:X_batch, \n",
    "                                                 model.targets:y_batch, \n",
    "                                                 model.keep_probs:1.0})\n",
    "    \n",
    "    test_accuracy.append(a)\n",
    "    ii += batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.984000027179718\n"
     ]
    }
   ],
   "source": [
    "print(\"Test accuracy: {}\".format(np.mean(test_accuracy)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A simple example on how to print predictions from this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_batch = X_test[0:0+batch_size]\n",
    "print_y_batch = y_test[0:0+batch_size].reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = session.run(model.preds, feed_dict={model.inputs:print_batch, \n",
    "                                                     model.targets:np.ones((50, 1)), \n",
    "                                                     model.keep_probs:1.0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "binarized_preds = np.where(predictions > 0.5, 1, 0)\n",
    "binarized_preds = binarized_preds.reshape(-1, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0,\n",
       "       0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1,\n",
       "       0, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarized_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweet:  For this you'll need int to vocab matching\n",
      "Prediction: 0\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(print_batch)):\n",
    "    print(\"Tweet: \", \"For this you'll need int to vocab matching\")\n",
    "    print(\"Prediction: {}\".format(binarized_preds[i]))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "session.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}