sEMG-LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sEMG LSTM training \n",
    "\n",
    "Based implementation on https://github.com/healthDataScience/deep-learning-HAR and modified to fit our dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from utilities import *\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in data and split into train/test portions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.abspath(os.getcwd())\n",
    "X_train, labels_train, list_ch_train, X_test, labels_test, list_ch_test = read_data(\n",
    "    data_path='%s%ssEMG%sDatabase 2%s' % (path, os.sep, os.sep, os.sep),\n",
    "    shuffle=True,\n",
    "    sub_split=True\n",
    ")\n",
    "\n",
    "assert list_ch_train == list_ch_test, 'Mismatch in channels!'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test = standardize(X_train, X_test) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train/validation split using sklearn utility. Manually specify random state seed to make testing more deterministic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tr, X_vld, lab_tr, lab_vld = train_test_split(\n",
    "    X_train,\n",
    "    labels_train, \n",
    "    stratify=labels_train,\n",
    "    random_state=123\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One-hot encoding:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_tr = one_hot(lab_tr)\n",
    "y_vld = one_hot(lab_vld)\n",
    "y_test = one_hot(labels_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize Tensorflow + Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "lstm_size = 6          # 3 times the amount of channels\n",
    "lstm_layers = 2        # Number of layers\n",
    "learning_rate = 0.0001\n",
    "batch_size = 100\n",
    "seq_len = 2496\n",
    "epochs = 1000\n",
    "\n",
    "# Fixed\n",
    "n_classes = 6\n",
    "n_channels = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct Graph\n",
    "Placeholders:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device('/gpu:0'):\n",
    "    graph = tf.Graph()\n",
    "\n",
    "    with graph.as_default():\n",
    "        inputs_ = tf.placeholder(tf.float32, [None, seq_len, n_channels], name='inputs')\n",
    "        labels_ = tf.placeholder(tf.float32, [None, n_classes], name='labels')\n",
    "        keep_prob_ = tf.placeholder(tf.float32, name='keep')\n",
    "        learning_rate_ = tf.placeholder(tf.float32, name='learning_rate')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Construct inputs to LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n"
     ]
    }
   ],
   "source": [
    "with tf.device('/gpu:0'):\n",
    "    with graph.as_default():\n",
    "        \n",
    "        # Construct the LSTM inputs and LSTM cells\n",
    "        lstm_in = tf.transpose(inputs_, [1, 0, 2]) # reshape into (seq_len, N, channels)\n",
    "        lstm_in = tf.reshape(lstm_in, [-1, n_channels]) # Now (seq_len*N, n_channels)\n",
    "\n",
    "        # To cells\n",
    "        lstm_in = tf.layers.dense(lstm_in, lstm_size, activation=None)\n",
    "\n",
    "        # Open up the tensor into a list of seq_len pieces\n",
    "        lstm_in = tf.split(lstm_in, seq_len, 0)\n",
    "\n",
    "        # Add LSTM layers\n",
    "        lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "        drop = tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=keep_prob_)\n",
    "        cell = tf.contrib.rnn.MultiRNNCell([drop] * lstm_layers)\n",
    "        initial_state = cell.zero_state(batch_size, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define forward pass, cost function and optimizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-a99942d1aed8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m             \u001b[0mlstm_in\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m             \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m             \u001b[0minitial_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m         )\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36mstatic_rnn\u001b[0;34m(cell, inputs, initial_state, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m   1328\u001b[0m             state_size=cell.state_size)\n\u001b[1;32m   1329\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1330\u001b[0;31m         \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1331\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1332\u001b[0m       \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m   1315\u001b[0m         \u001b[0mvarscope\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       \u001b[0;31m# pylint: disable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1317\u001b[0;31m       \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1318\u001b[0m       \u001b[0;31m# pylint: enable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m    189\u001b[0m         \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope_attrname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 191\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRNNCell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_rnn_get_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgetter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    712\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    713\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0min_deferred_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 714\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    715\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m             raise ValueError('A layer\\'s `call` method should return a Tensor '\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, state)\u001b[0m\n\u001b[1;32m   1240\u001b[0m                                       [-1, cell.state_size])\n\u001b[1;32m   1241\u001b[0m           \u001b[0mcur_state_pos\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1242\u001b[0;31m         \u001b[0mcur_inp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur_inp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcur_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1243\u001b[0m         \u001b[0mnew_states\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m   1056\u001b[0m                              \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_recurrent_input_noise\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1057\u001b[0m                              self._input_keep_prob)\n\u001b[0;32m-> 1058\u001b[0;31m     \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mscope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1059\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_should_dropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state_keep_prob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1060\u001b[0m       \u001b[0;31m# Identify which subsets of the state to perform dropout on and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope, *args, **kwargs)\u001b[0m\n\u001b[1;32m    296\u001b[0m     \u001b[0;31m# method.  See the class docstring for more details.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m     return base_layer.Layer.__call__(self, inputs, state, scope=scope,\n\u001b[0;32m--> 298\u001b[0;31m                                      *args, **kwargs)\n\u001b[0m\u001b[1;32m    299\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    300\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    712\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    713\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0min_deferred_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 714\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    715\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m             raise ValueError('A layer\\'s `call` method should return a Tensor '\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, state)\u001b[0m\n\u001b[1;32m    584\u001b[0m         value=gate_inputs, num_or_size_splits=4, axis=one)\n\u001b[1;32m    585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m     \u001b[0mforget_bias_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconstant_op\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forget_bias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m     \u001b[0;31m# Note that using `add` and `multiply` instead of `+` and `*` gives a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m     \u001b[0;31m# performance improvement. So using those at the cost of readability.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py\u001b[0m in \u001b[0;36mconstant\u001b[0;34m(value, dtype, shape, name, verify_shape)\u001b[0m\n\u001b[1;32m    212\u001b[0m   tensor_value.tensor.CopyFrom(\n\u001b[1;32m    213\u001b[0m       tensor_util.make_tensor_proto(\n\u001b[0;32m--> 214\u001b[0;31m           value, dtype=dtype, shape=shape, verify_shape=verify_shape))\n\u001b[0m\u001b[1;32m    215\u001b[0m   \u001b[0mdtype_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mattr_value_pb2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAttrValue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtensor_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m   const_tensor = g.create_op(\n",
      "\u001b[0;32m/virtualenvs/machine_learning/lib/python3.6/site-packages/tensorflow/python/framework/tensor_util.py\u001b[0m in \u001b[0;36mmake_tensor_proto\u001b[0;34m(values, dtype, shape, verify_shape)\u001b[0m\n\u001b[1;32m    470\u001b[0m   \u001b[0;31m# If shape is not given, get the shape from the numpy array.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    471\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 472\u001b[0;31m     \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnparray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    473\u001b[0m     \u001b[0mis_same_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m     \u001b[0mshape_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnparray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "with tf.device('/gpu:0'):\n",
    "    with graph.as_default():\n",
    "        outputs, final_state = tf.contrib.rnn.static_rnn(\n",
    "            cell,\n",
    "            lstm_in,\n",
    "            dtype=tf.float32,\n",
    "            initial_state=initial_state\n",
    "        )\n",
    "\n",
    "        logits = tf.layers.dense(outputs[-1], n_classes, name='logits')\n",
    "\n",
    "        # Cost and optimizer\n",
    "        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels_))\n",
    "\n",
    "        # Gradient clipping\n",
    "        train_op = tf.train.AdamOptimizer(learning_rate_)\n",
    "        gradients = train_op.compute_gradients(cost)\n",
    "        capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients]\n",
    "        optimizer = train_op.apply_gradients(capped_gradients)\n",
    "\n",
    "        # Accuracy\n",
    "        correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(labels_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (os.path.exists('checkpoints') == False):\n",
    "    !mkdir checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "validation_acc = []\n",
    "validation_loss = []\n",
    "\n",
    "train_acc = []\n",
    "train_loss = []\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    with graph.as_default():\n",
    "        saver = tf.train.Saver()\n",
    "\n",
    "    with tf.Session(graph=graph, config=tf.ConfigProto(log_device_placement=True)) as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        iteration = 1\n",
    "\n",
    "        for e in range(epochs):\n",
    "            # Initialize \n",
    "            state = sess.run(initial_state)\n",
    "\n",
    "            # Loop over batches\n",
    "            for x,y in get_batches(X_tr, y_tr, batch_size):\n",
    "                print(iteration)\n",
    "\n",
    "                # Feed dictionary\n",
    "                feed = {\n",
    "                    inputs_: x,\n",
    "                    labels_: y,\n",
    "                    keep_prob_: 0.5, \n",
    "                    initial_state: state,\n",
    "                    learning_rate_: learning_rate\n",
    "                }\n",
    "\n",
    "                loss, _, state, acc = sess.run(\n",
    "                    [cost, optimizer, final_state, accuracy], \n",
    "                    feed_dict = feed\n",
    "                )\n",
    "                train_acc.append(acc)\n",
    "                train_loss.append(loss)\n",
    "\n",
    "                # Print at each 5 iterations\n",
    "                if (iteration % 5 == 0):\n",
    "                    print(\n",
    "                        'Epoch: {}/{}'.format(e, epochs),\n",
    "                        'Iteration: {:d}'.format(iteration),\n",
    "                        'Train loss: {:6f}'.format(loss),\n",
    "                        'Train acc: {:.6f}'.format(acc)\n",
    "                    )\n",
    "\n",
    "                # Compute validation loss at every 25 iterations\n",
    "                if (iteration % 25 == 0):\n",
    "\n",
    "                    # Initiate for validation set\n",
    "                    val_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "\n",
    "                    val_acc_ = []\n",
    "                    val_loss_ = []\n",
    "                    for x_v, y_v in get_batches(X_vld, y_vld, batch_size):\n",
    "                        \n",
    "                        # Feed\n",
    "                        feed = {\n",
    "                            inputs_: x_v,\n",
    "                            labels_: y_v,\n",
    "                            keep_prob_: 1.0,\n",
    "                            initial_state : val_state\n",
    "                        }\n",
    "\n",
    "                        # Loss\n",
    "                        loss_v, state_v, acc_v = sess.run([cost, final_state, accuracy], feed_dict=feed)\n",
    "\n",
    "                        val_acc_.append(acc_v)\n",
    "                        val_loss_.append(loss_v)\n",
    "\n",
    "                    # Print info\n",
    "                    print(\n",
    "                        'Epoch: {}/{}'.format(e, epochs),\n",
    "                        'Iteration: {:d}'.format(iteration),\n",
    "                        'Validation loss: {:6f}'.format(np.mean(val_loss_)),\n",
    "                        'Validation acc: {:.6f}'.format(np.mean(val_acc_))\n",
    "                    )\n",
    "\n",
    "                    # Store\n",
    "                    validation_acc.append(np.mean(val_acc_))\n",
    "                    validation_loss.append(np.mean(val_loss_))\n",
    "\n",
    "                iteration += 1\n",
    "\n",
    "        saver.save(sess,'checkpoints/semg-lstm.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Plot training and test loss\n",
    "t = np.arange(iteration - 1)\n",
    "\n",
    "plt.figure(figsize=(6, 6))\n",
    "plt.plot(t, np.array(train_loss), 'r-', t[t % 25 == 0], np.array(validation_loss), 'b*')\n",
    "plt.xlabel('iteration')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend(['train', 'validation'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot Accuracies\n",
    "plt.figure(figsize=(6, 6))\n",
    "\n",
    "plt.plot(t, np.array(train_acc), 'r-', t[t % 25 == 0], validation_acc, 'b*')\n",
    "plt.xlabel('iteration')\n",
    "plt.ylabel('Accuray')\n",
    "plt.legend(['train', 'validation'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_acc = []\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    \n",
    "    # Restore checkpoint\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n",
    "    test_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "    \n",
    "    for x_t, y_t in get_batches(X_test, y_test, batch_size):\n",
    "        feed = {inputs_: x_t,\n",
    "                labels_: y_t,\n",
    "                keep_prob_: 1,\n",
    "                initial_state: test_state}\n",
    "        \n",
    "        batch_acc, test_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "        test_acc.append(batch_acc)\n",
    "    print('Test accuracy: {:.6f}'.format(np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}