notebooks/cnn_mnist_basic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MODERN CONVOLUTIONAL NEURAL NETWORK\n",
    "### DROPOUT + BATCH NORMALIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PACKAGES LOADED\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "%matplotlib inline  \n",
    "print (\"PACKAGES LOADED\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOAD MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('data/', one_hot=True)\n",
    "trainimg   = mnist.train.images\n",
    "trainlabel = mnist.train.labels\n",
    "testimg    = mnist.test.images\n",
    "testlabel  = mnist.test.labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SELECT DEVICE TO BE USED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "device_type = \"/gpu:1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEFINE CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN READY\n"
     ]
    }
   ],
   "source": [
    "n_input  = 784\n",
    "n_output = 10\n",
    "with tf.device(device_type):\n",
    "    weights  = {\n",
    "        'wc1': tf.Variable(tf.truncated_normal([3, 3, 1, 64], stddev=0.1)),\n",
    "        'wc2': tf.Variable(tf.truncated_normal([3, 3, 64, 128], stddev=0.1)),\n",
    "        'wd1': tf.Variable(tf.truncated_normal([7*7*128, 1024], stddev=0.1)),\n",
    "        'wd2': tf.Variable(tf.truncated_normal([1024, n_output], stddev=0.1))\n",
    "    }\n",
    "    biases   = {\n",
    "        'bc1': tf.Variable(tf.random_normal([64], stddev=0.1)),\n",
    "        'bc2': tf.Variable(tf.random_normal([128], stddev=0.1)),\n",
    "        'bd1': tf.Variable(tf.random_normal([1024], stddev=0.1)),\n",
    "        'bd2': tf.Variable(tf.random_normal([n_output], stddev=0.1))\n",
    "    }\n",
    "    def conv_basic(_input, _w, _b, _keepratio):\n",
    "        # INPUT\n",
    "        _input_r = tf.reshape(_input, shape=[-1, 28, 28, 1])\n",
    "        # CONV LAYER 1\n",
    "        _conv1 = tf.nn.conv2d(_input_r, _w['wc1'], strides=[1, 1, 1, 1], padding='SAME')\n",
    "        _mean, _var = tf.nn.moments(_conv1, [0, 1, 2])\n",
    "        _conv1 = tf.nn.batch_normalization(_conv1, _mean, _var, 0, 1, 0.0001)\n",
    "        _conv1 = tf.nn.relu(tf.nn.bias_add(_conv1, _b['bc1']))\n",
    "        _pool1 = tf.nn.max_pool(_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        _pool_dr1 = tf.nn.dropout(_pool1, _keepratio)\n",
    "        # CONV LAYER 2\n",
    "        _conv2 = tf.nn.conv2d(_pool_dr1, _w['wc2'], strides=[1, 1, 1, 1], padding='SAME')\n",
    "        _mean, _var = tf.nn.moments(_conv2, [0, 1, 2])\n",
    "        _conv2 = tf.nn.batch_normalization(_conv2, _mean, _var, 0, 1, 0.0001)\n",
    "        _conv2 = tf.nn.relu(tf.nn.bias_add(_conv2, _b['bc2']))\n",
    "        _pool2 = tf.nn.max_pool(_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        _pool_dr2 = tf.nn.dropout(_pool2, _keepratio)\n",
    "        # VECTORIZE\n",
    "        _dense1 = tf.reshape(_pool_dr2, [-1, _w['wd1'].get_shape().as_list()[0]])\n",
    "        # FULLY CONNECTED LAYER 1\n",
    "        _fc1 = tf.nn.relu(tf.add(tf.matmul(_dense1, _w['wd1']), _b['bd1']))\n",
    "        _fc_dr1 = tf.nn.dropout(_fc1, _keepratio)\n",
    "        # FULLY CONNECTED LAYER 2\n",
    "        _out = tf.add(tf.matmul(_fc_dr1, _w['wd2']), _b['bd2'])\n",
    "        # RETURN\n",
    "        out = { 'input_r': _input_r, 'conv1': _conv1, 'pool1': _pool1, 'pool1_dr1': _pool_dr1,\n",
    "            'conv2': _conv2, 'pool2': _pool2, 'pool_dr2': _pool_dr2, 'dense1': _dense1,\n",
    "            'fc1': _fc1, 'fc_dr1': _fc_dr1, 'out': _out\n",
    "        }\n",
    "        return out\n",
    "print (\"CNN READY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEFINE COMPUTATIONAL GRAPH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GRAPH READY\n"
     ]
    }
   ],
   "source": [
    "# PLACEHOLDERS\n",
    "x = tf.placeholder(tf.float32, [None, n_input])\n",
    "y = tf.placeholder(tf.float32, [None, n_output])\n",
    "keepratio = tf.placeholder(tf.float32)\n",
    "\n",
    "# FUNCTIONS\n",
    "with tf.device(device_type):\n",
    "    _pred = conv_basic(x, weights, biases, keepratio)['out']\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(_pred, y))\n",
    "    optm = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cost)\n",
    "    _corr = tf.equal(tf.argmax(_pred,1), tf.argmax(y,1)) \n",
    "    accr = tf.reduce_mean(tf.cast(_corr, tf.float32)) \n",
    "    init = tf.initialize_all_variables()\n",
    "    \n",
    "# SAVER\n",
    "save_step = 1\n",
    "saver = tf.train.Saver(max_to_keep=3)\n",
    "print (\"GRAPH READY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OPTIMIZE\n",
    "## DO TRAIN OR NOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "do_train = 1\n",
    "sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/015 cost: 0.707494674\n",
      " Training accuracy: 0.990\n",
      " Test accuracy: 0.962\n",
      "Epoch: 001/015 cost: 0.091100394\n",
      " Training accuracy: 0.970\n",
      " Test accuracy: 0.976\n",
      "Epoch: 002/015 cost: 0.062834177\n",
      " Training accuracy: 0.970\n",
      " Test accuracy: 0.982\n",
      "Epoch: 003/015 cost: 0.050124394\n",
      " Training accuracy: 0.980\n",
      " Test accuracy: 0.986\n",
      "Epoch: 004/015 cost: 0.041069326\n",
      " Training accuracy: 0.990\n",
      " Test accuracy: 0.985\n",
      "Epoch: 005/015 cost: 0.035412403\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.989\n",
      "Epoch: 006/015 cost: 0.030160291\n",
      " Training accuracy: 0.990\n",
      " Test accuracy: 0.991\n",
      "Epoch: 007/015 cost: 0.026010393\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.990\n",
      "Epoch: 008/015 cost: 0.024443544\n",
      " Training accuracy: 0.980\n",
      " Test accuracy: 0.986\n",
      "Epoch: 009/015 cost: 0.021579011\n",
      " Training accuracy: 0.980\n",
      " Test accuracy: 0.990\n",
      "Epoch: 010/015 cost: 0.018166464\n",
      " Training accuracy: 0.990\n",
      " Test accuracy: 0.990\n",
      "Epoch: 011/015 cost: 0.017664607\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.991\n",
      "Epoch: 012/015 cost: 0.015602452\n",
      " Training accuracy: 0.990\n",
      " Test accuracy: 0.990\n",
      "Epoch: 013/015 cost: 0.013644544\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.991\n",
      "Epoch: 014/015 cost: 0.011789304\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.993\n",
      "OPTIMIZATION FINISHED\n"
     ]
    }
   ],
   "source": [
    "training_epochs = 15\n",
    "batch_size      = 100\n",
    "display_step    = 1\n",
    "if do_train == 1:\n",
    "    for epoch in range(training_epochs):\n",
    "        avg_cost = 0.\n",
    "        total_batch = int(mnist.train.num_examples/batch_size)\n",
    "        # Loop over all batches\n",
    "        for i in range(total_batch):\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            # Fit training using batch data\n",
    "            sess.run(optm, feed_dict={x: batch_xs, y: batch_ys, keepratio:0.7})\n",
    "            # Compute average loss\n",
    "            avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keepratio:1.})/total_batch\n",
    "\n",
    "        # Display logs per epoch step\n",
    "        if epoch % display_step == 0: \n",
    "            print (\"Epoch: %03d/%03d cost: %.9f\" % (epoch, training_epochs, avg_cost))\n",
    "            train_acc = sess.run(accr, feed_dict={x: batch_xs, y: batch_ys, keepratio:1.})\n",
    "            print (\" Training accuracy: %.3f\" % (train_acc))\n",
    "            test_acc = sess.run(accr, feed_dict={x: testimg, y: testlabel, keepratio:1.})\n",
    "            print (\" Test accuracy: %.3f\" % (test_acc))\n",
    "\n",
    "        # Save Net\n",
    "        if epoch % save_step == 0:\n",
    "            saver.save(sess, \"nets/cnn_mnist_basic.ckpt-\" + str(epoch))\n",
    "\n",
    "    print (\"OPTIMIZATION FINISHED\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RESTORE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if do_train == 0:\n",
    "    epoch = training_epochs-1\n",
    "    saver.restore(sess, \"nets/cnn_mnist_basic.ckpt-\" + str(epoch))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# COMPUTE TEST ACCURACY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " TEST ACCURACY: 0.993\n"
     ]
    }
   ],
   "source": [
    "test_acc = sess.run(accr, feed_dict={x: testimg, y: testlabel, keepratio:1.})\n",
    "print (\" TEST ACCURACY: %.3f\" % (test_acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}