v4_Gesture_Recognition_Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AI DEMO - Summit -Tensorflow Dance Move Training\n",
    "\n",
    "This notebook should be able to run on the OCP4 Demo Cluster by simply selecting \"Cell\" and then \"Run All\" at the top of the page.\n",
    "\n",
    "Running this notebook will:\n",
    "* Pull gesture data from local repo\n",
    "* Preprocess the gesture data\n",
    "* Train a Tensorflow neural network model\n",
    "* Save the model for serving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:05:31.570796Z",
     "start_time": "2019-04-08T18:05:31.210702Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:05:33.341662Z",
     "start_time": "2019-04-08T18:05:31.573212Z"
    }
   },
   "outputs": [],
   "source": [
    "import s3fs\n",
    "import scipy.stats\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "from pathlib import Path\n",
    "from datetime import datetime as dt\n",
    "import matplotlib.pylab as plt\n",
    "import json\n",
    "import time\n",
    "import os\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix,classification_report\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder, MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Functions to read and preprocess raw data from Open Data Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:05:33.444906Z",
     "start_time": "2019-04-08T18:05:33.349351Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_raw_data(fs, path, debug=False):\n",
    "    '''Reads raw data from S3 - not parallelized\n",
    "    '''\n",
    "    data = {}\n",
    "    \n",
    "    gestures_paths = fs.ls(path)\n",
    "    \n",
    "    for gesture in gestures_paths:\n",
    "        files_to_read = fs.ls(gesture)\n",
    "    \n",
    "        key = gesture.split('/')[-1] \n",
    "        data[key] = []\n",
    "        if debug:\n",
    "            print(key)\n",
    "    \n",
    "        for file in files_to_read:\n",
    "            with fs.open(file) as f:\n",
    "                content = json.loads(f.read())\n",
    "            data[key].append(content)\n",
    "    \n",
    "    return data\n",
    "\n",
    "counter = 0\n",
    "def process_crosses_data(data_crosses):\n",
    "    '''Returns list of size N_samples and each element = tuple(index, label, acc+rot vel dataframe, rot angle dataframe)\n",
    "    '''\n",
    "\n",
    "    global counter\n",
    "    \n",
    "    data_cross_list = []\n",
    "    gesture = 'draw-cross'\n",
    "\n",
    "    for d in data_crosses: #loop over each sample\n",
    "        acc_data = pd.DataFrame([x['acceleration'] for elem in data_crosses[d] for x in elem['motion']])\n",
    "        acc_data.columns = ['acceleration_x', 'acceleration_y', 'acceleration_z']\n",
    "\n",
    "        rot_data = pd.DataFrame([x['rotationRate'] for elem in data_crosses[d] for x in elem['motion']])\n",
    "        rot_data.columns = ['rotation_alpha', 'rotation_beta', 'rotation_gamma']\n",
    "\n",
    "        timestamp_data = pd.DataFrame([x['timestamp'] for elem in data_crosses[d] for x in elem['motion']])\n",
    "        timestamp_data.columns = ['timestamp']\n",
    "\n",
    "        motion_df = pd.concat([acc_data, rot_data, timestamp_data], axis=1).sort_values('timestamp', ascending=True).drop('timestamp', axis=1)\n",
    "        orientation_df = None\n",
    "\n",
    "        data_cross_list.append((counter, gesture, motion_df, orientation_df))\n",
    "        counter += 1\n",
    "        \n",
    "    return data_cross_list\n",
    "\n",
    "def process_dance_data(data_dance_raw):\n",
    "    '''Returns list of size N_samples and each element = tuple(index, label, acc+rot vel dataframe, rot angle dataframe)\n",
    "    '''\n",
    "\n",
    "    #Ugly\n",
    "    #convert appropriate data to dataframes\n",
    "    global counter\n",
    "    \n",
    "    data = []\n",
    "    #counter = 0 #counter from before\n",
    "    for k in data_dance_raw.keys(): #loop over each dance type\n",
    "        d = data_dance_raw[k]\n",
    "        for example in d: #loop over each sample\n",
    "            gesture = example['gesture']\n",
    "\n",
    "            motion = np.array(example['motion'])\n",
    "            orientation = np.array(example['orientation'])\n",
    "\n",
    "            #process motion data\n",
    "            if len(motion)>0:\n",
    "                if motion.shape[1]==7: #incl. acc and rotational velocities\n",
    "                    motion_df = pd.DataFrame(motion, columns=['acceleration_x', 'acceleration_y', 'acceleration_z',\n",
    "                                                             'rotation_alpha', 'rotation_beta', 'rotation_gamma',\n",
    "                                                              'timestamp'\n",
    "                                                             ])\\\n",
    "                                    .sort_values('timestamp', ascending=True)\\\n",
    "                                    .drop('timestamp', axis=1)\n",
    "                \n",
    "                elif motion.shape[1]==4: #incl. acc only\n",
    "                    motion_df = pd.DataFrame(motion, columns=['acceleration_x', 'acceleration_y', 'acceleration_z',\n",
    "                                                              'timestamp'\n",
    "                                                             ])\\\n",
    "                                    .sort_values('timestamp', ascending=True)\\\n",
    "                                    .drop('timestamp', axis=1)\n",
    "\n",
    "                else: #shouldn't enter\n",
    "                    motion_df = pd.DataFrame()\n",
    "            else: #no motion data recorded\n",
    "                motion_df = pd.DataFrame()\n",
    "\n",
    "\n",
    "            #process orientation data\n",
    "            if len(orientation) > 0:\n",
    "                orientation_df = pd.DataFrame(orientation, columns=['alpha', 'beta', 'gamma', 'timestamp'])\\\n",
    "                                        .sort_values('timestamp', ascending=True)\\\n",
    "                                        .drop('timestamp', axis=1)\n",
    "            else:\n",
    "                orientation_df = pd.DataFrame()\n",
    "\n",
    "            data.append((counter, gesture, motion_df, orientation_df))\n",
    "\n",
    "            counter += 1\n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read raw dance and crosses data from local repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dance_raw = json.loads(Path('data/dance/data_dance_raw.json').read_text())\n",
    "data_cross_raw = json.loads(Path('data/dance/data_cross_raw.json').read_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:05:54.143657Z",
     "start_time": "2019-04-08T18:05:50.603518Z"
    }
   },
   "outputs": [],
   "source": [
    "counter = 0\n",
    "data_dance_proc = process_dance_data(data_dance_raw)\n",
    "data_cross_proc = process_crosses_data(data_cross_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:07.028199Z",
     "start_time": "2019-04-08T18:06:06.865281Z"
    }
   },
   "outputs": [],
   "source": [
    "del data_dance_raw\n",
    "del data_cross_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:07.074812Z",
     "start_time": "2019-04-08T18:06:07.033016Z"
    }
   },
   "outputs": [],
   "source": [
    "data = data_dance_proc + data_cross_proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:07.129117Z",
     "start_time": "2019-04-08T18:06:07.078535Z"
    }
   },
   "outputs": [],
   "source": [
    "del data_dance_proc\n",
    "del data_cross_proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:07.193965Z",
     "start_time": "2019-04-08T18:06:07.140974Z"
    }
   },
   "outputs": [],
   "source": [
    "data = [d for d in data if d[2].shape[1]==6 and d[2].shape[0]>50] # <---- clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:07.249879Z",
     "start_time": "2019-04-08T18:06:07.200085Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['draw-circle', 'draw-cross', 'draw-triangle', 'fever', 'floss',\n",
       "        'roll', 'shake'], dtype='<U13'),\n",
       " array([121, 185, 128, 123, 110, 137, 136]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique([d[1] for d in data], return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:07.311322Z",
     "start_time": "2019-04-08T18:06:07.252483Z"
    }
   },
   "outputs": [],
   "source": [
    "def featurize(ts, bins, TAG):\n",
    "    '''Take time-series and create features\n",
    "    '''\n",
    "    mean = np.mean(ts)\n",
    "    median = np.median(ts)\n",
    "    std = np.std(ts)\n",
    "    length = len(ts)\n",
    "    kurtosis = scipy.stats.kurtosis(ts)\n",
    "    \n",
    "    n,b = np.histogram(ts, bins=bins)\n",
    "    n = np.array(n)/float(np.sum(n)) #normalize i.e. fraction of entries in each bin\n",
    "    \n",
    "    if median == 0: \n",
    "        features = {f'{TAG}_mean_over_median': 0, #dimensionless            \n",
    "                    f'{TAG}_std_over_median': 0, #dimensionless            \n",
    "                    f'{TAG}_length': length,\n",
    "                    f'{TAG}_kurtosis': kurtosis, #already dimensionless by definition\n",
    "                   }\n",
    "        \n",
    "    else: \n",
    "        features = {f'{TAG}_mean_over_median': mean/median, #dimensionless            \n",
    "            f'{TAG}_std_over_median': std/median, #dimensionless            \n",
    "            f'{TAG}_length': length,\n",
    "            f'{TAG}_kurtosis': kurtosis, #already dimensionless by definition\n",
    "           }\n",
    "        \n",
    "    for i, val in enumerate(n):\n",
    "        features[f'{TAG}_binfrac_{i}'] = val\n",
    "        \n",
    "    \n",
    "    return features\n",
    "\n",
    "def find_bins(ts_list, method='freedman'):\n",
    "    ''' Find bin edges for histograms based on different methods\n",
    "    '''\n",
    "    \n",
    "    ts_all = np.concatenate(ts_list)\n",
    "    \n",
    "    plt.clf()\n",
    "    if method in ['freedman', 'scott', 'knuth', 'blocks']:\n",
    "        n,b = np.histogram(ts_all, bins=method)\n",
    "        plt.hist(ts_all, bins=b)\n",
    "    else:\n",
    "        n,b,p = plt.hist(ts_all)\n",
    "    \n",
    "    return ts_all, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:07.357668Z",
     "start_time": "2019-04-08T18:06:07.314981Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_dataframe(data, cols):\n",
    "    df_list = []\n",
    "    for col in cols:\n",
    "        index_list, feature_list, label_list = [], [], []\n",
    "        for d in data:\n",
    "            features = featurize(d[2][col], bins=col_bins[col], TAG=col.upper())\n",
    "\n",
    "            feature_list.append(features)\n",
    "            index_list.append(d[0])\n",
    "            label_list.append(d[1])\n",
    "\n",
    "        feature_col_df = pd.DataFrame(feature_list)\n",
    "\n",
    "        df_list.append(feature_col_df)\n",
    "        \n",
    "    df = pd.concat(df_list, axis=1)\n",
    "    df['index'] = index_list\n",
    "    df['label'] = label_list\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:08.150820Z",
     "start_time": "2019-04-08T18:06:07.359786Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGDFJREFUeJzt3X+s3fV93/Hna/agSTaCgVvKbJidxclm0KqCFzx1rdKQGUO6mmkkMtqGl1qx1kDXTa0S0/xBFYIEXTdWNEJFg4eJIhzG0mI1Zq5LyKJJM2BCAhhCuTFJuBZgBxvYlgXq5L0/zsfp6eVc36/vub7nGp4P6eh+v+/v5/v9vo/zTV75/jjnpKqQJKmLvzbqBiRJJw5DQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqbOFo25gtp1xxhm1dOnSUbchSSeURx555PtVNTbduDddaCxdupTdu3ePug1JOqEk+W6XcV6ekiR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSps2lDI8nmJPuTPDGp/utJvpVkT5Lf7atfk2Q8ydNJLu6rr2m18SSb+urLkjzY6l9MclKrn9zmx9vypbPxhiVJM9flTOMOYE1/IckvAWuBn62qc4Hfa/UVwDrg3LbOZ5MsSLIAuAW4BFgBXNHGAtwI3FRV7wYOARtafQNwqNVvauMkSSM0bWhU1deAg5PKvwbcUFWvtTH7W30tsLWqXquqZ4Fx4H3tNV5Ve6vqdWArsDZJgA8A97T1twCX9W1rS5u+B7iojZckjchMPxH+HuAXklwP/BD4rap6GFgM7OobN9FqAM9Nql8InA68XFWHB4xffGSdqjqc5JU2/vsz7FkaqaWbvjyyfX/nhg+NbN96c5lpaCwETgNWAf8AuDvJu2atq2OUZCOwEeCcc84ZVRuS9KY306enJoAvVc9DwI+BM4B9wNl945a02lT1l4BTkyycVKd/nbb8nW38G1TVbVW1sqpWjo1N+31bkqQZmmlo/DHwSwBJ3gOcRO+y0TZgXXvyaRmwHHgIeBhY3p6UOonezfJtVVXAA8DlbbvrgXvb9LY2T1v+lTZekjQi016eSnIX8H7gjCQTwLXAZmBzewz3dWB9+x/0PUnuBp4EDgNXVdWP2nauBnYAC4DNVbWn7eKTwNYknwEeBW5v9duBzycZp3cjft0svF9J0hCmDY2qumKKRf9iivHXA9cPqG8Htg+o76X3dNXk+g+BD0/XnyRp7viJcElSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ9OGRpLNSfa3n3advOw3k1SSM9p8ktycZDzJY0nO7xu7Pskz7bW+r35BksfbOjcnSauflmRnG78zyaLZecuSpJnqcqZxB7BmcjHJ2cBq4Ht95UuA5e21Ebi1jT2N3m+LX0jvp12v7QuBW4GP9a13ZF+bgPurajlwf5uXJI3QtKFRVV8DDg5YdBPwCaD6amuBO6tnF3BqkrOAi4GdVXWwqg4BO4E1bdkpVbWrqgq4E7isb1tb2vSWvrokaURmdE8jyVpgX1V9c9KixcBzffMTrXa0+sSAOsCZVfV8m34BOHMmvUqSZs/CY10hyduB36Z3aWpOVFUlqamWJ9lI73IY55xzzly1JUlvOTM50/g7wDLgm0m+AywBvp7kZ4B9wNl9Y5e02tHqSwbUAV5sl69of/dP1VBV3VZVK6tq5djY2AzekiSpi2MOjap6vKp+uqqWVtVSepeUzq+qF4BtwJXtKapVwCvtEtMOYHWSRe0G+GpgR1v2apJV7ampK4F72662AUeeslrfV5ckjUiXR27vAv4X8N4kE0k2HGX4dmAvMA78IfBxgKo6CFwHPNxen2412pjPtXW+DdzX6jcA/zjJM8AH27wkaYSmvadRVVdMs3xp33QBV00xbjOweUB9N3DegPpLwEXT9SdJmjt+IlyS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1FmXn3vdnGR/kif6av8+ybeSPJbkj5Kc2rfsmiTjSZ5OcnFffU2rjSfZ1FdfluTBVv9ikpNa/eQ2P96WL52tNy1JmpkuZxp3AGsm1XYC51XV3wf+HLgGIMkKYB1wblvns0kWJFkA3AJcAqwArmhjAW4EbqqqdwOHgCO/Qb4BONTqN7VxkqQRmjY0quprwMFJtT+tqsNtdhewpE2vBbZW1WtV9SwwDryvvcaram9VvQ5sBdYmCfAB4J62/hbgsr5tbWnT9wAXtfGSpBGZjXsavwrc16YXA8/1LZtotanqpwMv9wXQkfpf2VZb/kob/wZJNibZnWT3gQMHhn5DkqTBhgqNJJ8CDgNfmJ12ZqaqbquqlVW1cmxsbJStSNKb2sKZrpjkXwG/DFxUVdXK+4Cz+4YtaTWmqL8EnJpkYTub6B9/ZFsTSRYC72zjJUkjMqMzjSRrgE8Av1JVP+hbtA1Y1558WgYsBx4CHgaWtyelTqJ3s3xbC5sHgMvb+uuBe/u2tb5NXw58pS+cJEkjMO2ZRpK7gPcDZySZAK6l97TUycDOdm96V1X966rak+Ru4El6l62uqqofte1cDewAFgCbq2pP28Unga1JPgM8Ctze6rcDn08yTu9G/LpZeL+SpCFMGxpVdcWA8u0DakfGXw9cP6C+Hdg+oL6X3tNVk+s/BD48XX+SpLnjJ8IlSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSepsxr/cJ52olm768qhbkE5YnmlIkjozNCRJnU0bGkk2J9mf5Im+2mlJdiZ5pv1d1OpJcnOS8SSPJTm/b531bfwzSdb31S9I8nhb5+a034+dah+SpNHpcqZxB7BmUm0TcH9VLQfub/MAlwDL22sjcCv0AoDeb4tfSO+nXa/tC4FbgY/1rbdmmn1IkkZk2tCoqq8BByeV1wJb2vQW4LK++p3Vsws4NclZwMXAzqo6WFWHgJ3AmrbslKraVVUF3DlpW4P2IUkakZne0zizqp5v0y8AZ7bpxcBzfeMmWu1o9YkB9aPt4w2SbEyyO8nuAwcOzODtSJK6GPpGeDtDqFnoZcb7qKrbqmplVa0cGxs7nq1I0lvaTEPjxXZpifZ3f6vvA87uG7ek1Y5WXzKgfrR9SJJGZKahsQ048gTUeuDevvqV7SmqVcAr7RLTDmB1kkXtBvhqYEdb9mqSVe2pqSsnbWvQPiRJIzLtJ8KT3AW8HzgjyQS9p6BuAO5OsgH4LvCRNnw7cCkwDvwA+ChAVR1Mch3wcBv36ao6cnP94/Se0HobcF97cZR9SJJGZNrQqKorplh00YCxBVw1xXY2A5sH1HcD5w2ovzRoH5Kk0fET4ZKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0NFRpJ/l2SPUmeSHJXkp9KsizJg0nGk3wxyUlt7MltfrwtX9q3nWta/ekkF/fV17TaeJJNw/QqSRrejEMjyWLg3wArq+o8YAGwDrgRuKmq3g0cAja0VTYAh1r9pjaOJCvaeucCa4DPJlmQZAFwC3AJsAK4oo2VJI3IsJenFgJvS7IQeDvwPPAB4J62fAtwWZte2+Zpyy9KklbfWlWvVdWz9H5f/H3tNV5Ve6vqdWBrGytJGpEZh0ZV7QN+D/gevbB4BXgEeLmqDrdhE8DiNr0YeK6te7iNP72/PmmdqeqSpBEZ5vLUInr/z38Z8LeAd9C7vDTnkmxMsjvJ7gMHDoyiBUl6Sxjm8tQHgWer6kBV/QXwJeDngVPb5SqAJcC+Nr0POBugLX8n8FJ/fdI6U9XfoKpuq6qVVbVybGxsiLckSTqaYULje8CqJG9v9yYuAp4EHgAub2PWA/e26W1tnrb8K1VVrb6uPV21DFgOPAQ8DCxvT2OdRO9m+bYh+pUkDWnh9EMGq6oHk9wDfB04DDwK3AZ8Gdia5DOtdntb5Xbg80nGgYP0QoCq2pPkbnqBcxi4qqp+BJDkamAHvSezNlfVnpn2K0ka3oxDA6CqrgWunVTeS+/Jp8ljfwh8eIrtXA9cP6C+Hdg+TI+SpNnjJ8IlSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0NFRpJTk1yT5JvJXkqyT9MclqSnUmeaX8XtbFJcnOS8SSPJTm/bzvr2/hnkqzvq1+Q5PG2zs3tt8glSSMy7JnG7wP/var+LvCzwFPAJuD+qloO3N/mAS4BlrfXRuBWgCSn0fvJ2Avp/UzstUeCpo35WN96a4bsV5I0hBmHRpJ3Ar8I3A5QVa9X1cvAWmBLG7YFuKxNrwXurJ5dwKlJzgIuBnZW1cGqOgTsBNa0ZadU1a6qKuDOvm1JkkZgmDONZcAB4L8keTTJ55K8Azizqp5vY14AzmzTi4Hn+tafaLWj1ScG1CVJIzJMaCwEzgduraqfA/4vf3kpCoB2hlBD7KOTJBuT7E6y+8CBA8d7d5L0ljVMaEwAE1X1YJu/h16IvNguLdH+7m/L9wFn962/pNWOVl8yoP4GVXVbVa2sqpVjY2NDvCVJ0tHMODSq6gXguSTvbaWLgCeBbcCRJ6DWA/e26W3Ale0pqlXAK+0y1g5gdZJF7Qb4amBHW/ZqklXtqakr+7YlSRqBhUOu/+vAF5KcBOwFPkoviO5OsgH4LvCRNnY7cCkwDvygjaWqDia5Dni4jft0VR1s0x8H7gDeBtzXXpKkERkqNKrqG8DKAYsuGjC2gKum2M5mYPOA+m7gvGF6lCTNHj8RLknqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqbOjQSLIgyaNJ/qTNL0vyYJLxJF9sPwVLkpPb/HhbvrRvG9e0+tNJLu6rr2m18SSbhu1VkjSc2TjT+A3gqb75G4GbqurdwCFgQ6tvAA61+k1tHElWAOuAc4E1wGdbEC0AbgEuAVYAV7SxkqQRGSo0kiwBPgR8rs0H+ABwTxuyBbisTa9t87TlF7Xxa4GtVfVaVT0LjAPva6/xqtpbVa8DW9tYSdKIDHum8Z+ATwA/bvOnAy9X1eE2PwEsbtOLgecA2vJX2vif1CetM1X9DZJsTLI7ye4DBw4M+ZYkSVOZcWgk+WVgf1U9Mov9zEhV3VZVK6tq5djY2KjbkaQ3rYVDrPvzwK8kuRT4KeAU4PeBU5MsbGcTS4B9bfw+4GxgIslC4J3AS331I/rXmaouSRqBGZ9pVNU1VbWkqpbSu5H9lar658ADwOVt2Hrg3ja9rc3Tln+lqqrV17Wnq5YBy4GHgIeB5e1prJPaPrbNtF9J0vCGOdOYyieBrUk+AzwK3N7qtwOfTzIOHKQXAlTVniR3A08Ch4GrqupHAEmuBnYAC4DNVbXnOPQrSepoVkKjqr4KfLVN76X35NPkMT8EPjzF+tcD1w+obwe2z0aPkqTh+YlwSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzmYcGknOTvJAkieT7EnyG61+WpKdSZ5pfxe1epLcnGQ8yWNJzu/b1vo2/pkk6/vqFyR5vK1zc5IM82YlScMZ5kzjMPCbVbUCWAVclWQFsAm4v6qWA/e3eYBL6P3+93JgI3Ar9EIGuBa4kN4v/l17JGjamI/1rbdmiH4lSUOacWhU1fNV9fU2/b+Bp4DFwFpgSxu2BbisTa8F7qyeXcCpSc4CLgZ2VtXBqjoE7ATWtGWnVNWuqirgzr5tSZJGYFbuaSRZCvwc8CBwZlU93xa9AJzZphcDz/WtNtFqR6tPDKhLkkZk6NBI8jeA/wb826p6tX9ZO0OoYffRoYeNSXYn2X3gwIHjvTtJessaKjSS/HV6gfGFqvpSK7/YLi3R/u5v9X3A2X2rL2m1o9WXDKi/QVXdVlUrq2rl2NjYMG9JknQUwzw9FeB24Kmq+o99i7YBR56AWg/c21e/sj1FtQp4pV3G2gGsTrKo3QBfDexoy15Nsqrt68q+bUmSRmDhEOv+PPAvgceTfKPVfhu4Abg7yQbgu8BH2rLtwKXAOPAD4KMAVXUwyXXAw23cp6vqYJv+OHAH8DbgvvaSJI3IjEOjqv4nMNXnJi4aML6Aq6bY1mZg84D6buC8mfYoSZpdfiJcktSZoSFJ6szQkCR1ZmhIkjob5ukpSSeIpZu+PJL9fueGD41kvzp+PNOQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzH7nVSIzqEVBJw/FMQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ3N+9BIsibJ00nGk2wadT+S9FY2r0MjyQLgFuASYAVwRZIVo+1Kkt665vvnNN4HjFfVXoAkW4G1wJMj7UpSJ6P8PI5fy358zPfQWAw81zc/AVw4ol7elPyQnaRjMd9Do5MkG4GNbfb/JHl6lP0cxRnA90fdxDE4kfq11+PnROr3J73mxhF3Mr359u/6t7sMmu+hsQ84u29+Sav9FVV1G3DbXDU1U0l2V9XKUffR1YnUr70ePydSv/Z6/M3rG+HAw8DyJMuSnASsA7aNuCdJesua12caVXU4ydXADmABsLmq9oy4LUl6y5rXoQFQVduB7aPuY5bM+0tok5xI/drr8XMi9Wuvx1mqatQ9SJJOEPP9noYkaR4xNGZRkg8n2ZPkx0lW9tWXJvl/Sb7RXn/Qt+yCJI+3r0m5OUla/bQkO5M80/4umote27JrWj9PJ7m4rz7wK13agwoPtvoX20MLx02S30myr+/f89KZ9j7X5ksf/ZJ8px2D30iyu9UGHn/pubn1/1iS8+egv81J9id5oq92zP0lWd/GP5Nk/Rz2esIerwNVla9ZegF/D3gv8FVgZV99KfDEFOs8BKwCAtwHXNLqvwtsatObgBvnqNcVwDeBk4FlwLfpPYSwoE2/CzipjVnR1rkbWNem/wD4teP87/w7wG8NqB9z73N8fMyLPgb09R3gjEm1gccfcGk7TtOO2wfnoL9fBM7v/+/QsfYHnAbsbX8XtelFc9TrCXm8TvXyTGMWVdVTVdX5g4VJzgJOqapd1TuK7gQua4vXAlva9Ja++vHudS2wtapeq6pngXF6X+fyk690qarXga3A2nZm9AHgnuPV6zE4pt5H0N986aOLqY6/tcCd1bMLOLUdx8dNVX0NODhkfxcDO6vqYFUdAnYCa+ao16nM9+N1IENj7ixL8miS/5HkF1ptMb2vRjliotUAzqyq59v0C8CZc9TnoK9uWXyU+unAy1V1eFL9eLu6XX7Y3Hfp7lh7n2vzpY/JCvjTJI+k9+0KMPXxN1/ew7H2N+q+T8TjdaB5/8jtfJPkz4CfGbDoU1V17xSrPQ+cU1UvJbkA+OMk53bdZ1VVkmN+zG2Gvc4LR+sduBW4jt7/2F0H/AfgV+euuzedf1RV+5L8NLAzybf6F870+Jsr870/3mTHq6FxjKrqgzNY5zXgtTb9SJJvA++h95UoS/qG9n9NyotJzqqq59vp9f656JWjf3XLoPpL9C4BLGxnGwO/6uVYde09yR8Cf9Jmj7X3udbpa3HmWlXta3/3J/kjepdHpjr+5st7ONb+9gHvn1T/6hz0SVW9eGT6BDteB/Ly1BxIMpbeb4OQ5F3AcmBvO71+Ncmqdm/gSuDIGcA24MgTHuv76sfbNmBdkpOTLGu9PsQUX+nS7sU8AFw+V71Ouob+T4EjT6ocU+/Hs8cpzJc+fiLJO5L8zSPTwGp6/55THX/bgCvbU0qrgFf6LhPNpWPtbwewOsmidnlodasddyfw8TrYqO/Ev5le9A6ICXpnFS8CO1r9nwF7gG8AXwf+Sd86K+kdRN8G/jN/+YHL04H7gWeAPwNOm4te27JPtX6epj3N1eqXAn/eln2qr/4uegf7OPBfgZOP87/z54HHgcfo/ZfprJn2PoJjZF70Mek/u2+2154jPU11/NF7KumW1v/j9D15dxx7vIveJd6/aMfshpn0R++S0Hh7fXQOez1hj9dBLz8RLknqzMtTkqTODA1JUmeGhiSpM0NDktSZoSFJ6szQkCR1ZmhIkjozNCRJnf1/2XYYOclliowAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cols = ['acceleration_x', 'acceleration_y', 'acceleration_z',\n",
    "       'rotation_alpha', 'rotation_beta', 'rotation_gamma']\n",
    "method = 'plt'\n",
    "\n",
    "col_bins = {}\n",
    "\n",
    "for col in cols:\n",
    "    ts_all, b = find_bins([d[2][col] for d in data], method=method)\n",
    "    col_bins[col] = b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:12.595862Z",
     "start_time": "2019-04-08T18:06:08.156415Z"
    }
   },
   "outputs": [],
   "source": [
    "df = create_dataframe(data, cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:12.644305Z",
     "start_time": "2019-04-08T18:06:12.597478Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ACCELERATION_X_binfrac_0                     0\n",
       "ACCELERATION_X_binfrac_1                     0\n",
       "ACCELERATION_X_binfrac_2                     0\n",
       "ACCELERATION_X_binfrac_3                     0\n",
       "ACCELERATION_X_binfrac_4               0.22884\n",
       "ACCELERATION_X_binfrac_5              0.761755\n",
       "ACCELERATION_X_binfrac_6            0.00940439\n",
       "ACCELERATION_X_binfrac_7                     0\n",
       "ACCELERATION_X_binfrac_8                     0\n",
       "ACCELERATION_X_binfrac_9                     0\n",
       "ACCELERATION_X_kurtosis               0.184926\n",
       "ACCELERATION_X_length                      319\n",
       "ACCELERATION_X_mean_over_median        3.48315\n",
       "ACCELERATION_X_std_over_median        -27.2695\n",
       "ACCELERATION_Y_binfrac_0                     0\n",
       "ACCELERATION_Y_binfrac_1                     0\n",
       "ACCELERATION_Y_binfrac_2                     0\n",
       "ACCELERATION_Y_binfrac_3                     0\n",
       "ACCELERATION_Y_binfrac_4              0.990596\n",
       "ACCELERATION_Y_binfrac_5            0.00940439\n",
       "ACCELERATION_Y_binfrac_6                     0\n",
       "ACCELERATION_Y_binfrac_7                     0\n",
       "ACCELERATION_Y_binfrac_8                     0\n",
       "ACCELERATION_Y_binfrac_9                     0\n",
       "ACCELERATION_Y_kurtosis               -0.32075\n",
       "ACCELERATION_Y_length                      319\n",
       "ACCELERATION_Y_mean_over_median        3.45779\n",
       "ACCELERATION_Y_std_over_median         17.0681\n",
       "ACCELERATION_Z_binfrac_0                     0\n",
       "ACCELERATION_Z_binfrac_1                     0\n",
       "                                      ...     \n",
       "ROTATION_BETA_binfrac_0                      0\n",
       "ROTATION_BETA_binfrac_1                      0\n",
       "ROTATION_BETA_binfrac_2                      0\n",
       "ROTATION_BETA_binfrac_3                      0\n",
       "ROTATION_BETA_binfrac_4               0.253918\n",
       "ROTATION_BETA_binfrac_5               0.746082\n",
       "ROTATION_BETA_binfrac_6                      0\n",
       "ROTATION_BETA_binfrac_7                      0\n",
       "ROTATION_BETA_binfrac_8                      0\n",
       "ROTATION_BETA_binfrac_9                      0\n",
       "ROTATION_BETA_kurtosis               -0.167105\n",
       "ROTATION_BETA_length                       319\n",
       "ROTATION_BETA_mean_over_median        0.589452\n",
       "ROTATION_BETA_std_over_median          8.80727\n",
       "ROTATION_GAMMA_binfrac_0                     0\n",
       "ROTATION_GAMMA_binfrac_1                     0\n",
       "ROTATION_GAMMA_binfrac_2                     0\n",
       "ROTATION_GAMMA_binfrac_3                     0\n",
       "ROTATION_GAMMA_binfrac_4              0.316614\n",
       "ROTATION_GAMMA_binfrac_5              0.683386\n",
       "ROTATION_GAMMA_binfrac_6                     0\n",
       "ROTATION_GAMMA_binfrac_7                     0\n",
       "ROTATION_GAMMA_binfrac_8                     0\n",
       "ROTATION_GAMMA_binfrac_9                     0\n",
       "ROTATION_GAMMA_kurtosis               0.248245\n",
       "ROTATION_GAMMA_length                      319\n",
       "ROTATION_GAMMA_mean_over_median        10.4361\n",
       "ROTATION_GAMMA_std_over_median        -42.2039\n",
       "index                                       20\n",
       "label                              draw-circle\n",
       "Name: 0, Length: 86, dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:12.706152Z",
     "start_time": "2019-04-08T18:06:12.653433Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "draw-cross     185\n",
       "roll           137\n",
       "shake          136\n",
       "fever          123\n",
       "draw-circle    121\n",
       "floss          110\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[~df['label'].isin(['draw-triangle'])]\n",
    "df['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:12.756583Z",
     "start_time": "2019-04-08T18:06:12.708415Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/macermak/.local/share/virtualenvs/2019-demo4-ai-FZCiEg3v/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "\n",
    "train_df, test_df = train_test_split(df, train_size=0.70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:12.808256Z",
     "start_time": "2019-04-08T18:06:12.759375Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size = (568, 86)\n",
      "test size = (244, 86)\n",
      "draw-cross     131\n",
      "shake          103\n",
      "fever           91\n",
      "draw-circle     84\n",
      "roll            82\n",
      "floss           77\n",
      "Name: label, dtype: int64\n",
      "roll           55\n",
      "draw-cross     54\n",
      "draw-circle    37\n",
      "floss          33\n",
      "shake          33\n",
      "fever          32\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(f'train size = {train_df.shape}')\n",
    "print(f'test size = {test_df.shape}')\n",
    "\n",
    "print(train_df['label'].value_counts())\n",
    "print(test_df['label'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:12.878218Z",
     "start_time": "2019-04-08T18:06:12.810617Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(train_labels, train_pred):\n",
    "    fig = plt.figure(figsize=(8,8))\n",
    "    ax = plt.subplot()\n",
    "\n",
    "    labels = np.sort(list(train_labels.value_counts().index))\n",
    "\n",
    "    confusion = confusion_matrix(train_labels, train_pred, labels=labels)\n",
    "    ax.matshow(confusion)\n",
    "\n",
    "    ax.set_xticks(range(len(labels)))\n",
    "    ax.set_yticks(range(len(labels)))\n",
    "\n",
    "    ax.set_xticklabels(labels, rotation=90);\n",
    "    ax.set_yticklabels(labels);\n",
    "\n",
    "    for i in range(len(labels)):\n",
    "        for j in range(len(labels)):        \n",
    "            ax.text(j, i, confusion[i,j], va='center', ha='center')\n",
    "\n",
    "    plt.xlabel('predicted')    \n",
    "    plt.ylabel('true')\n",
    "    \n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:12.929805Z",
     "start_time": "2019-04-08T18:06:12.881171Z"
    }
   },
   "outputs": [],
   "source": [
    "train_label = np.array(train_df['label'])\n",
    "test_label = np.array(test_df['label'])\n",
    "\n",
    "label_encoder = LabelEncoder()\n",
    "one_hot_encoder = OneHotEncoder(sparse=False,categories='auto')\n",
    "\n",
    "train_integer = label_encoder.fit_transform(train_label)\n",
    "train_onehot = one_hot_encoder.fit_transform(train_integer.reshape(len(train_integer), 1))\n",
    "\n",
    "test_integer = label_encoder.transform(test_label)\n",
    "test_onehot = one_hot_encoder.transform(test_integer.reshape(len(test_integer), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:19.192022Z",
     "start_time": "2019-04-08T18:06:18.981722Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/macermak/.local/share/virtualenvs/2019-demo4-ai-FZCiEg3v/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/macermak/.local/share/virtualenvs/2019-demo4-ai-FZCiEg3v/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "model = keras.Sequential([\n",
    "    keras.layers.Dense(30, activation=tf.nn.sigmoid, input_shape=(85,)),\n",
    "    keras.layers.Dropout(0.05, seed=42),\n",
    "    keras.layers.Dense(6, activation=tf.nn.log_softmax)\n",
    "]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:19.424151Z",
     "start_time": "2019-04-08T18:06:19.194946Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.Adam(lr=1e-3)\n",
    "loss      = tf.nn.softmax_cross_entropy_with_logits_v2\n",
    "metrics   = ['accuracy']\n",
    "\n",
    "model.compile(optimizer=optimizer,  loss=loss, metrics=metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:30.532917Z",
     "start_time": "2019-04-08T18:06:19.426123Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 1.2183 - acc: 0.6150\n",
      "Epoch 2/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.7586 - acc: 0.8752\n",
      "Epoch 3/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.5390 - acc: 0.9172\n",
      "Epoch 4/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.4089 - acc: 0.9364\n",
      "Epoch 5/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.3390 - acc: 0.9401\n",
      "Epoch 6/15\n",
      "200/200 [==============================] - 1s 5ms/step - loss: 0.2968 - acc: 0.9416\n",
      "Epoch 7/15\n",
      "200/200 [==============================] - 1s 4ms/step - loss: 0.2615 - acc: 0.9455\n",
      "Epoch 8/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.2273 - acc: 0.9478\n",
      "Epoch 9/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.2072 - acc: 0.9545\n",
      "Epoch 10/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.1910 - acc: 0.9587\n",
      "Epoch 11/15\n",
      "200/200 [==============================] - 1s 5ms/step - loss: 0.1819 - acc: 0.9606\n",
      "Epoch 12/15\n",
      "200/200 [==============================] - 1s 4ms/step - loss: 0.1709 - acc: 0.9630\n",
      "Epoch 13/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.1657 - acc: 0.9629\n",
      "Epoch 14/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.1582 - acc: 0.9648\n",
      "Epoch 15/15\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 0.1528 - acc: 0.9654\n"
     ]
    }
   ],
   "source": [
    "history = model.fit( train_df.drop('label', axis=1), train_onehot, epochs=15, steps_per_epoch=200, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:30.794702Z",
     "start_time": "2019-04-08T18:06:30.539474Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "244/244 [==============================] - 0s 708us/sample - loss: 0.4797 - acc: 0.9016\n",
      "Test accuracy: 0.90163934\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = model.evaluate(test_df.drop('label', axis=1), test_onehot)\n",
    "\n",
    "print('Test accuracy:', test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:31.646369Z",
     "start_time": "2019-04-08T18:06:30.809047Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'predicted')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAFqCAYAAAA6M7CZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcXHWd7vHPU70nTSedhZCEQDZMlIABAhEkbqOALG6guIy7g+jF5c4F9boy6iiDemcGmEFxAVRmRkEQRhkREzFBImFJIKACL2TPQjbS6aS3dH3vH3WCTeilOnT3qV/yvF+vfqXPqdNVTx+Kp0/9zqaIwMzM0lHIO4CZmQ2Oi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9v2eZImSfq+pP/Jpl8i6YN55zLri4vbhpSkCyU1SaqRtFjSBkl/m3euAVwB3ARMyaYfBD6ZWxqzAbi4baidEBEtwKnAo8Bs4LxcEw1sQkT8FCgCRMROoDvfSGZ9c3HbUKvO/j0FuDoituYZpkzbJY0HAkDSy4AUcts+qnrgRcwG5ReS/gy0AR+RNBFozznTQP4euAGYJen3wETgjHwjmfVNvqyrDTVJ44CtEdEtaRTQFBHr8s7VH0nVwBxAwAMR0ZVzJLM+eYvbhpSktwK/ykr788CRwFeBiituSW/p46EXSSIirh3RQGZlcnHbUPtCRFwt6XjgtcA3gEuBhfnG6tVp/TwWgIvbKpKHSmxISVoZEUdI+jqwOiL+Y9e8vLOZ7S18VIkNtackfQc4E7hRUh0V/j6T9DVJY3tMN0v6ap6ZzPrjLW4bUtnOyJMobW0/JGkycFhE/DrnaH3q7ROBpLsj4si8Mpn1p6K3hCw9EbEDeBg4UdI5wP6VXNqZquyTAQCSGoC6fpY3y5WL24aUpE8AVwH7Z18/lvSxfFMN6CpgsaQPZtcouRm4MudMZn3yUIkNKUn3AsdGxPZsejSwPCIOzzdZ/ySdROkoGICbI+KmPPOY9ceHA9pQE8+9zkd3Nq8iSaoCfhMRrwZ+lXces3K4uG2oXQ7cLum6bPpNwPdzzNOv7EShoqQxiVxXxcxDJTb0JB0JHJ9NLouIlXnmGYik64EjKI1tb981PyI+nlsos364uG3IZMMO90fE3LyzDIak9/Y2PyK8g9IqkodKbMhkww4PSDooIh7PO0+5XNCWGhe3DbVm4H5JK3jusMMb8ovUO0k/jYi3SVpNdi3unir9SBjbd3moxIaUpFf2Nj8ifjfSWQYiaXJErJV0cG+PR8RjI53JrBze4rah9jiwNiLa4dmzECflG6l3EbE2+7ZAIpnNwGdO2tC7muzejZnubF4lSzGz7cNc3DbUqiOic9dE9n1tjnnKkWJm24e5uG2obZD07I5ISW8ENuaYpxwpZrZ9mHdO2pCSNIvSRZumUDrV/Qng3RHxcK7B+pFi5tRJGpVdSdL2gIvbhoWkRoCIaM07S7lSzJwaSccB3wMaI+IgSS8FPhwRH805WlI8VGLDIiu//8o7x2CkmDlB/wycCGwCiIh7gFfkmihBLm4bTlPzDrAHUsyclIh4YrdZ3b0uaH1ycdtwquiLS/UhxcwpeSIbLglJNZLOBf6Ud6i+SKqS9Nu8c+zOxW1DStLfZCewEBEfyDtPOVLMnLCzgf9F6ZPNU8D8bLoiRUQ3UJQ0Ju8sPXnnpA0pSVcCxwKbgWXAUuDWiNiSa7B+pJg5VZLGRcTm3ebNiIhH8so0kEq87K+L24aFpCnAGcC5wJSIqPjLK6SUOTuE8cmI6JD0KuBw4IcR8Uy+yfon6ffA6yOiJZt+MXB1RMzLN1nfKvGyvy5uG1KS/hZYBBxG6SSWWyndTGF5rsH6kWjmVcACYDpwI3A9cGhEnJxnroFIOgX4FHAKMAf4IfCuiFiVa7ABZENpB0XEA3lnARe3DTFJG4GHgW8Dv42IR/NNNLBEM98dEUdKOg9oj4iLJa2MiCPyzjYQSW+iVN77AadHxIM5R+qXpNOAbwK1ETFD0nzgy3leqrhiPwpamiJigqRDKR2b+4+SDgEeiIh35xytTylmBrokvQN4L3BaNq8mxzz9knQxz73m+RhKfyzPkVTpt4k7HzgGuAUgIlZJmplnIBe3DSlJTcBBwMGUPsaP4blX3qs4KWYG3k/pCI1/jIhHJM0AfpRzpv7cudv0Xbmk2DNdEbFVUs95ub4/PFRiQ0rSvZTGiG8FlkbEkzlHGlCKmXuS1AxMi4h7886yN5L0fWAx8BngdODjQE1EnJ1bJhe3WXok3QK8gdKn5ruAp4HfR8Tf55lrINkw1NeBlwD1u+ZHRK5DD/2RNAr4HHBCNusm4Ku7bryRSyYXtw0lSRMp7Xg6lOf+j/ma3EININHMKyPiCEkforS1/SVJ91b6fTIl3Qp8idI1S06jNORTiIgv5hqsH5KOjIi7887Rk8+crECStklq6eVrm6SWvPMN4Crgz8AM4B+AR4E78gxUhhQzV0uaDLwN+EXeYQahISIWU9pofCwizqd0aGAl+5akP0n6iqSKON7cxV2BImK/iGjq5Wu/iGjKO98AxkfE9ynt0Plddgp5xW65ZlLM/GVKH9kfjog7sqMcHso5Uzk6JBWAhySdI+nNQGPeofoTEa8GXg1sAL4jabWkz+eZyUMlFUjSuP4e3/2U4Uoi6Q8R8TJJNwEXAWuAayJiVs7R+pRi5lRJOprSRaXGAl+hdATPhRHxh1yDlUnSYZSG1c6MiNxub+firkCSHqF0zKt6eTgqfEfOqZSu9zENuBhoAv4hIm7INVg/Es18IKWsL89mLQM+kdoRMSnITss/k9LlEDYCPwF+FhFP55bJxW1DRVIV8PGI+Oe8s5QrxcwAkm4G/oO/Hrv9t5ROHX9dfqkGJulFwHmUjpl/9jySCt8RvJzSDTaujog1eecBF3fFy25iu+sOIbdEREXviJK0IiKOyTvHYCSaeVVEzB9oXqWRdA+lSwvcRY8bKERERZ+QU2nXKvGZkxVM0gXA0ZSOegD4hKTjIuKzOcYayO8lXULp42TPS2BW1OFUu0kx86bs4lj/mU2/g+x2YBVuZ0RcmneIweh5rRKgIq5V4i3uCpad0Tc/IorZdBWwspKP1e1xt5BdbyxRGpev5I/CKWY+mNIY97GUct9Gacjn8VyD9aHHDvePUzpZ6DqgY9fjFb7D/S5KRxndsusiXpJWR8RheWXyFnflG0vpAv9Q2gNfkSTtOmPvFzx/x2pFbh2kmHmXiHiM0pmTqbiL567j83juOq7YHe70fq2SXN8fLu4KpdK75JvAymyLUJTGuj+Ta7C+7Zf9O4fS8M71lDKfBqzIK9QAksss6aL+Hq/Uq+xFxAwASW8DfhURLZK+ABxJ6bDASna/pHcCVdkp+x+n9AknNx4qqWCSVlO6PsLR2awVEbEux0gDkrQUOCUitmXT+wG/jIhX9P+T+Ukps6QnKV03oxl43q3V8rwrSzl2nZYv6XhKhf1N4IsRsTDnaM8j6UcR8W5JnwVGU/p/UZROfPpKntcq8RZ3ZbsbOLCSjyfuxSSgs8d0ZzavkqWUuYXSvQ//B3gVvR/rX8l2HUlyCvDdiPilpK/mGagfR2W3szuT0pmT3+rx2CjAxW29Wgi8S9JjlI522LXTrGJ3TlK6FdUKSddl028CrsgvTllSyvxtSpcYnclzr2ktSuOulTxWDPCUpO8ArwP+SVIdlXvpjZ7ruuf1xHNf1x4qqWDZkQPPk+2YqliSjqR0D0coXd96ZZ55ypFaZkmXRsRH8s4xWNklUk8CVkfEQ9mFsg6LiF/nHK1PlbiuXdxmZomp1I8oZmbWBxd3IiSdlXeGPZFibmceGSlmhsrI7eJOR+5vlj2UYm5nHhkpZoYKyO3iNjNLjHdODrEx46rigKk1Q/68z2zuZuy4qiF/XoB1fxq+G5B0FtupLdQPvOAeiOLwvHe7op0aDU9mhun/ty46qKFuWJ5b1cNz1HBnsY3aQsOwPPdwGq7cbd3b6Cy2lXVcvo/jHmIHTK3h0ht6PYqvYl14xPF5R9gjxY6OgReqMJFg5qpxE/OOsE9Yvvmaspf1UImZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYY30ihQnV3Bx994+OMn1TN174/lYjgB9/axO9u3EZVlTjtXWN4y/ua847Zq7ZiK6t3LKUz2gGYVjuHg+sOzTnVwO7vWs6G7qeoVT3H1Z2ad5yybYx1PMgqgmAqM5iuuXlH6ld37GTFlusp0k1EkQPqZjK78Zi8Y/Wr0jK7uAcg6VFgQURslNQaEcN3n68err38GQ6aVcv21iIAN13Twoa1XVzxm+kUCmLLxp0jEWOPFCgwt+EYmqomsDO6WN56PeOrp9BYVZl/aHaZUjWTaVVzuK/rtryjlC0ieICVHMEi6hnFChYzIabQqKa8o/WpQBVHj30D1YUaitHNii0/Z0LXQYytOSDvaH2qtMweKgFUUjHrYsPaLm7/bSsnnznm2Xk3XPUM7/7YeAqF0i3pmidU7t/cusIomqomAFCtGkYXxtJe3JFzqoE1FyZRQ23eMQZlK5tpoJFRaqSgApOYxgbW5B2rX5KoLpTuyxoUKVIEyrrVYm4qLXPl/t8/zCRNB24CbgeOAi6UdC6l/xq/jIhP55Xt376ygbM+M5Ed24vPzlvzeBe3/HIbt97UypjxVZzzxf05cEbll0xbcRvbujcxttr3LRwOHbRRz19vXFtPA1vZnGOi8kQUWb7lGnZ0b2VawzzG1kzKO9KAKilzxWxl5uQQ4N+B1wFfAV4DzAeOlvSmPAItX9xK8/gqXnTYc+8y3tUZ1NSJS284mFPOHMM3Pr0+j3iDsjO6WLV9CXMbFlKtyv8jYyNHKnDcuLfxyvHvYWvX02zbuSnvSAOqpMz77BZ35rGI+IOkNwK3RMQGAElXAa8Afl7Ok0g6CzgLYP8pL2yV3n9XG7ct3s7tt/yFzo5gR2uRr/3vtUw8oJpFJ+4HwPEnNvKNT1V2cRejyKodS5hcO4tJNdPzjrPXqqOBdtqenW6njboeW+CVrqZQx7jaqWzsfIL9qsfnHacslZB5X9/i3j4UTxIRl0XEgohYMHZc1Qt6rg99aiI/uW0m/7FsJp+/aDLzjx3FZ/95Mi9/XSOrlpfGie+5vY0DZ9QMRfRhERHc37aM0YUxTK+bl3ecvVoTzbTRSltspxhF1vMEE5mcd6x+dRbb6Cp2AKWjNTZ1PsHoqrE5p+pfpWXe17e4d1kBXCRpArAFeAdwcb6RnusdHxnH1z65jp/9YAv1owv8nwsqdw/8M93rWdP1MI2FZm7bVvrQckj9UUysmZZzsv7d23krW4rr6aKDpe3XMqv6cKZWz847Vr8KKjAn5rOSZQTBFKbTqDED/2COOoo7WN2yhIgiEEyqn83+ddPzjtWvSsusiMjtxfOU7Zz8RUTMy6bfAXyW3XZODvZwwDmH1celNxw8nNGH3IVHHJ93hD1S7OjIO8KgRYKZqyZ6x/JIWL75GrZ2PV3WoSr77BZ3RDwKzOsx/Z/Af/ay3PQe34/IMdxmZv3Z18e4zcyS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwSs8/eAWe4rL1vFF+fdXjeMQblpjVL846wR06asTDvCPuGBG+3BtDd0pJ3hEGJ2Fn2st7iNjNLjIvbzCwxLm4zs8S4uM3MEuPiNjNLjIvbzCwxLm4zs8S4uM3MEuPiNjNLjIvbzCwxLm4zs8S4uM3MEuPiNjNLjIvbzCwxLm4zs8S4uM3MEuPiNjNLjIvbzCwxvnVZAjbGOh5kFUEwlRlM19y8I/Vq5tGPsl9jgaoqqK4SK26axtX/3cqXv7mZPz3UyR9uPJAF8+vzjtmn+7uWs6H7KWpVz3F1p+Ydp2ypvD92aSu2snrHUjqjHYBptXM4uO7QnFMNrJLW84gWt6TzgdaI+OZIvm7KIoIHWMkRLKKeUaxgMRNiCo1qyjtarxZfM5UJ46uenZ43p5Zrvn8AH/nU0zmmKs+UqplMq5rDfV235R2lbKm9PwAKFJjbcAxNVRPYGV0sb72e8dVTaKxqzjtanyptPec+VCJpxP54jORrDZWtbKaBRkapkYIKTGIaG1iTd6yyvfhFtcyZXZt3jLI0FyZRQxpZd0nx/VFXGEVT1QQAqlXD6MJY2os7ck7Vv0pbz8Ne3JI+J+lBSbcCc7J5t0j6F0l3Ap+QdJqk2yWtlPQbSZOy5VZLGquSTZLek83/oaTX9fJas7Ofv0fS3ZJmSXqVpGWSbgD+mC3395Luy74+mc0bLemX2c/eJ+nMbP4Fkv4o6V5JI/5JoYM26ml4drqeBjpoG+kYZZHgpLev4egTnuCyH23NO84+IaX3R2/aitvY1r2JsdUT847Sr0pbz8O6BSrpKODtwPzste4G7soero2IBdlyzcDLIiIkfQj4FPB/gN8DLwceA/4CLAJ+CBwLfKSXl7wKuCAirpNUT+kP0zTgSGBeRDySZXo/sBAQcLuk3wEzgTURcUqWaYyk8cCbgblZtrF9/J5nAWcB1DNqz1bWXmDp9QcydXI1T2/cyYlnrmHu7FpecWzDwD9o+6Sd0cWq7UuY27CQaqX1SSdvw73FvQi4LiJ2REQLcEOPx37S4/sDgZskrQbOA3btqVgGvCL7uhQ4TNJUYEtEbO/5QpL2A6ZGxHUAEdEeEbs+f62IiEey74/PMm2PiFbg2iznauB1kv5J0qKI2ApsBdqB70t6C9Dr57mIuCwiFkTEghrqBrmK+ldHA+09/rK300YdlVmGUyeXtgP2n1DNm14/mjtWteecaO+X0vujp2IUWbVjCZNrZzGpZnrecQZUaes5zzHunsV7MXBJRBwGfBjYdejBUkqlugi4BdgAnEGp0JF0uaRVkm4cxGv1KiIepLRlvhr4qqQvRsRO4BjgGuBU4Ffl/WpDp4lm2milLbZTjCLreYKJTB7pGAPavqPIttbis9/f/Ls2Dp3jrajhlsr7o6eI4P62ZYwujGF63by845Sl0tbzcO+sWwpcIenr2WudBnynl+XGAE9l379318yIeELSBErDKn/JxsnPBc7JHn9/zyeR9KSkN0XEzyXVAVU837Is0wWUhkreDLxb0hRgc0T8WNIzwIckNQKjIuJGSb+nNFwzogoqMCfms5JlBMEUptOoMSMdY0DrN3Rz+gfWArBzJ7zjzY2c9JrRXHdjK5/4/AY2bOrmtHev5aWH1vKr/5qac9re3dt5K1uK6+mig6Xt1zKr+nCmVs/OO1a/Unl/9PRM93rWdD1MY6GZ27b9HIBD6o9iYs20nJP1rdLW87AWd0TcLeknwD3A08AdfSx6PnC1pC3AEmBGj8du568FvAz4OnBrH8/zbuA7kr4MdAFv7SPTFcCKbNb3ImKlpBOBb0gqZj/7EWA/4PpsvFzA3w/4Sw+DCZrMhArfipp5cA0rFx/0vPlvPrmRN5/cmEOiwTu89vi8I+yRFN4fPTVXH8CJYz6Qd4xBq6T1rIjIO8NepUnjYqH+Ju8Yg3LTmlV5R9gjJ81YmHeEQYuOjrwjDFpVU+UeE96f7paWvCMMyu2xmJbYrHKWzf04bjMzGxwXt5lZYlzcZmaJcXGbmSXGxW1mlhgXt5lZYlzcZmaJcXGbmSXGxW1mlhgXt5lZYlzcZmaJcXGbmSXGxW1mlhgXt5lZYlzcZmaJcXGbmSXGxW1mlpjhvuekJeDEKfPzjrBHtGRC3hEGTa/fmHeEQUvtTjL7Am9xm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxnfAScDGWMeDrCIIpjKD6Zqbd6QBpZC5u3Mnd37iGoqd3UR3kUmvnM2s9x9LRPDw95ez/ncPoYI48A2Hc9DplXmXoPu7lrOh+ylqVc9xdafmHadsKbw/dldJmUesuCWdD7RGxDdH8DWnABdFxBmD+JkrgF9ExDXDFmwQIoIHWMkRLKKeUaxgMRNiCo1qyjtan1LJXKip4qj/9xaqG2op7uzmjo9dzfiF09n+2Gban97GcVe+BxVE55YdeUft05SqmUyrmsN9XbflHaVsqbw/eqq0zLkOlUga1j8cEbGmt9Ie7tcdSlvZTAONjFIjBRWYxDQ2sCbvWP1KJbMkqhtqAYidRaK7iBBP3rCame9diAoCoLZ5VJ4x+9VcmEQNtXnHGJRU3h89VVrmYS1uSZ+T9KCkW4E52bxbJP2LpDuBT0g6TdLtklZK+o2kSdlyqyWNVckmSe/J5v9Q0ut6ea3Z2c/fI+luSbMkTZd0X/b4+yTdIGkJsDib9+nsde6RdEEvz3mUpN9JukvSTZImD9vK6kMHbdTT8Ox0PQ100DbSMQYlpczRXWT5h67id2/+LuOPOogxLzmAtjVbWffbB/nDh/+Tuz/9c7Y/uSXvmHuVlN4fu1Ra5mErbklHAW8H5gMnA0f3eLg2IhZExLeAW4GXRcQRwH8Bn8qW+T3wcuBQ4C/Aomz+sUBvnwuvAv4tIl4KHAes7WWZI4EzIuKVkl4PvBFYmP3MhbvlrwEuzpY/CvgB8I99/K5nSbpT0p1ddPS5TqzyqKrAsd97F4uu/iBb/7ye1kc2UuzsplBbzcu+8w6mnjKPP174m7xjmj3HcA4ZLAKui4gdAJJu6PHYT3p8fyDwk2xrthZ4JJu/DHgF8BhwKXCWpKnAlojY3vOFJO0HTI2I6wAioj2bv3ummyNic/b9a4HLd+XrMX+XOcA84Obsearo/Y8BEXEZcBlAk8ZFb8vsqToaaO/xl72dNup6/OWvRClmrmmso3n+gWxc8Rh1ExuZtGgWAPsvmsUfL7w553R7lxTfH5WWud8tbkn/nQ0v9Pr1Al63Z/FeDFwSEYcBHwbqs/lLKZX/IuAWYANwBqVCR9LlklZJunEPX3cgAu6PiPnZ12ERccIgfn5INNFMG620xXaKUWQ9TzCRER+xGZRUMnc+s4Ou1tInpO6OnWy+63FGH9TM/sfPZPPKJwHYcs9TjDpwbJ4x9zqpvD96qrTMA21x7zoC5C3AAcCPs+l3AOsH+NmlwBWSvp69zmnAd3pZbgzwVPb9e3fNjIgnJE2gNKzyl2yc/FzgnOzx9/d8EklPSnpTRPxcUh2lLeT+3Ax8UdJVEbFD0rjdtrofACZKOjYilmdDJy+KiPsHeN4hVVCBOTGflSwjCKYwnUaNGckIg5ZK5o5N27n/gpuJYpEowqRXHcLEY2cy9rCp3PfVX/H4NSupaqjhJee+Nu+ofbq381a2FNfTRQdL269lVvXhTK2enXesfqXy/uip0jL3W9wR8TsASd+KiAU9HvrvbOdifz97t6SfAPcATwN39LHo+cDVkrYAS4AZPR67nb8W8DLg65TGxHvzbuA7kr4MdAFvBYr95PuVpPnAnZI6gRuBz/Z4vFPSGcBFksZQWlf/AoxocQNM0GQmVPgWye5SyLzfrIm87LvvfN78msY6jrjgjTkkGrzDa4/PO8IeSeH9sbtKyqyIgYdkJf0JOCUi/pJNzwBujIgXD3O+5DRpXCzU3+QdY5+gJVPzjjB4r9+Yd4JBiw7vcB8Jt8diWmLz83bM9abcnZP/G7hF0l8ojf0eTGk82szMRlhZxZ0NKxwC7DrH888R4T/DZmY5KOs4bkmjgPOAcyLiHuAgSelcGMHMbC9S7gk4lwOdlE5+gdJRIF8dlkRmZtavcot7VkRcSOloDbKTVsoaRDczs6FVbnF3SmoAAkDSLPC53WZmeSj3qJLzgV8B0yRdRekaIu/v9yfMzGxYlHtUya8l3QW8jNIQySciIr0DUs3M9gLlHlWyOCI2RcQvI+IXEbFR0uLhDmdmZs/X7xa3pHpgFDBBUjN/3SHZBCR42pqZWfoGGir5MPBJYApwF38t7hbgkmHMZWZmfRjoIlP/CvyrpI9FxMUjlMnMzPpR7uGARUnPXpRYUrOkjw5TJjMz60e5xf13EfHMromI2AL83fBEMjOz/pRb3FXqcR8wSVWQ2K2lzcz2EuWegPMrSveF3HUHmw9n88zMbISVW9yfplTWH8mmbwa+NyyJzMysX+WeOVmkdKf1S4c3TvpUXU3VuIl5xxiUYktL3hH2SLzmqYEXqjAPXn5U3hEG7UXvvyvvCLabgU7A+WlEvE3SarILTPUUEYcPWzIzM+vVQFvcn8j+9U0TzMwqxEAn4KzN/n1sZOKYmdlABhoq2UYvQyS7RETTkCcyM7N+DbTFvR+ApK8Aa4EfUbpeybuAycOezszMnqfcE3DeEBH/HhHbIqIlIi4F3jicwczMrHflFvd2Se+SVCWpIOldwPbhDGZmZr0rt7jfCbwNWJ99vTWbZ2ZmI6zcE3AexUMjZmYVodxbl71I0mJJ92XTh0v6/PBGMzOz3pQ7VPJd4P8CXQARcS/w9uEKZWZmfSu3uEdFxIrd5u0c6jBmZjawcot7o6RZZCfjSDqD0nHdZmY2wsq9rOv/Ai4D5kp6CniE0kk4ZmY2wgYsbkkFYEFEvFbSaKAQEduGP5qZmfVmwKGS7Frcn8q+3+7SNjPLV7lj3L+RdK6kaZLG7foa1mRmZtarcse4z6S0Y/Kju82fObRxbHfdsZMVW66nSDcRRQ6om8nsxmPyjtWv+7uWs6H7KWpVz3F16VzKfWOs40FWEQRTmcF0zc070vPs3PQMm757Nd0trQgY/apjaDrh5XQ+vpbNV15HdHRSNb6ZCWefSaGhPu+4fUphXe+ukjKXW9wvoVTax1Mq8GXAt4cr1FCR9HFK98lsAq6LiHNyjjRoBao4euwbqC7UUIxuVmz5ORO6DmJszQF5R+vTlKqZTKuaw31dt+UdpWwRwQOs5AgWUc8oVrCYCTGFRlXWlYtVVaD57SdTO30qxbYO1p27Yk8gAAAQEklEQVR/MQ2Hzmbz5T9j7JknUz93Jq1L76TlxqWMPf2EvOP2KpV13VOlZS53qORK4MXARcDFlIr8yuEKNYQ+CrwO+FzeQfaUJKoLNQAERYoUKV1Zt3I1FyZRQ23eMQZlK5tpoJFRaqSgApOYxgbW5B3rearGNlE7fSoAhYY6aqbsz84tLXSt20jdnBkA1B86mx133Z9nzH6lsq57qrTM5Rb3vIj4UET8Nvv6O2DecAZ7oSR9m9JQzv8AzT3mT5e0RNK92Wn8B2Xz3yrpPkn3SFqazTtU0gpJq7LlD8njd4koctvmn/LbjVcwvvZAxtZMyiPGXq2DNuppeHa6ngY6aMsx0cB2bthC52NrqJs1jZqpk2i7+48A7LhjNd2bn8k5Xd9SXNeVlrnc4r5b0st2TUhaCNw5PJGGRkScDawBXg1s6fHQxcCV2Y2Or6L0KQLgi8CJEfFS4A3ZvLOBf42I+cAC4MmRyL47qcBx497GK8e/h61dT7Nt56Y8YlgFKbZ3sOGSH9P8zlMpNNQz/gOn07rkD6z90sVEeweqqso7og2jcse4jwJuk/R4Nn0Q8MCuu78ndrf3Y4G3ZN//CLgw+/73wBWSfgpcm81bDnxO0oHAtRHxUG9PKOks4CyA+kLjcOWmplDHuNqpbOx8gv2qxw/b6+yL6migvccWVDtt1PXYwqoksbObjZdcxehj5zNqQemDb82U/dn/vA8C0LVuA233PJBnxH6ltK53qbTM5W5xnwTMAF6Zfc3I5p0KnDY80UZWtoX+eWAacJek8RHxH5S2vtuAGyW9po+fvSwiFkTEgtrC0P7H7Cy20VXsAEpHmGzqfILRVWOH9DUMmmimjVbaYjvFKLKeJ5hYgXfniwg2/eBn1EyeSNNJi56d393SWnq8WGTrDb+l8dUL84o4oFTWdU+Vlrnc63HvTXd5v43SlQ1/ROm0/WUAkmZFxO3A7ZJeD0yTNAb4S0RclI2FHw4sGcmwHcUdrG5ZQuk8qGBS/Wz2r5s+khEG7d7OW9lSXE8XHSxtv5ZZ1YcztXp23rH6VVCBOTGflSwjCKYwnUaNyTvW83Q89Bg7bltJzYEHsPYLpVG+sWecQNf6TbQuXg7AqKPmMXrRUXnG7Fcq67qnSsusiD5v4p48SY9SGps+ldJp++dIOhi4HJgAbADeHxGPS7oWOITSIRuLgU8CnwbeTelytuuAd0bE5v5ec0zN/nHsuDOG6TcaHsWWlrwj7JHo6Mg7wqA9eHnlFmpfXvT+u/KOsE+4PRbTEpvLOmSs3DHuJEXE9OzbK7KvXZ8enjfkERFv2X0ecEH2ZWZWMcod4zYzswrh4jYzS4yL28wsMS5uM7PEuLjNzBLj4jYzS4yL28wsMS5uM7PEuLjNzBLj4jYzS4yL28wsMS5uM7PEuLjNzBLj4jYzS4yL28wsMS5uM7PEuLjNzBKzV98BJxdRhMRuqVWoq8s7wh7pTmw9Q5q3AUvxdmuQ5roul7e4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4zvgVLi2YiurdyylM9oBmFY7h4PrDs05Vf9SzAywMdbxIKsIgqnMYLrm5h2pLCnk3rnpGTZ992q6W1oRMPpVx9B0wsvpfHwtm6+8jujopGp8MxPOPpNCQ33ecXtVSet5ryxuSY8CCyJiY5nLvy9b/pzhzLUnChSY23AMTVUT2BldLG+9nvHVU2isas47Wp9SzBwRPMBKjmAR9YxiBYuZEFNoVFPe0fqVSm5VFWh++8nUTp9Ksa2DdedfTMOhs9l8+c8Ye+bJ1M+dSevSO2m5cSljTz8h77jPU2nr2UMlFa6uMIqmqgkAVKuG0YWxtBd35Jyqfylm3spmGmhklBopqMAkprGBNXnHGlAquavGNlE7fSoAhYY6aqbsz84tLXSt20jdnBkA1B86mx133Z9nzD5V2npOvrgljZb0S0n3SLpP0pnZQx+TdLek1VLpM42kYyQtl7RS0m2S5vTyfKdky0yQNFHSzyTdkX29fER/ud20FbexrXsTY6sn5hljUFLJ3EEb9TQ8O11PAx205ZioPCnm3rlhC52PraFu1jRqpk6i7e4/ArDjjtV0b34m53S9q7T1nHxxAycBayLipRExD/hVNn9jRBwJXAqcm837M7AoIo4Avgh8recTSXoz8Bng5GyY5V+Bf46Io4HTge8N+2/Th53RxartS5jbsJBq1eYVY1BSzGzDq9jewYZLfkzzO0+l0FDP+A+cTuuSP7D2SxcT7R2oqirviEnYG8a4VwPfkvRPwC8iYpkkgGuzx+8C3pJ9Pwa4UtIhQAA1PZ7nNcAC4ISIaMnmvRZ4SfZ8AE2SGiOitWcASWcBZwHUa/RQ/m4AFKPIqh1LmFw7i0k104f8+YdDapnraKC9xxZUO23U9djCqlQp5Y6d3Wy85CpGHzufUQvmAVAzZX/2P++DAHSt20DbPQ/kGbFPlbaek9/ijogHgSMpFfhXJX0xe6gj+7ebv/6B+grw22zL/DSg5+7rh4H9gBf1mFcAXhYR87OvqbuXdpbhsohYEBELagtDu0c8Iri/bRmjC2OYXjdvSJ97uKSYuYlm2milLbZTjCLreYKJTM471oBSyR0RbPrBz6iZPJGmkxY9O7+7pfS/UxSLbL3htzS+emFeEftVaes5+S1uSVOAzRHxY0nPAB/qZ/ExwFPZ9+/b7bHHgPOAayW9NSLuB34NfAz4RvZa8yNi1VDmH8gz3etZ0/UwjYVmbtv2cwAOqT+KiTXTRjLGoKSYuaACc2I+K1lGEExhOo0ak3esAaWSu+Ohx9hx20pqDjyAtV+4CICxZ5xA1/pNtC5eDsCoo+YxetFRecbsU6WtZ0VEbi8+FCSdSKlYi0AX8BHgGrLDASUtAL4ZEa+SdCxwJbAd+CXwtxExvefhgJKOAK6itEW+Ffg34MWU/sgtjYiz+8szpnpCHNv4xuH4VW033S0tAy9kL9iDl1dmmQ7kRe+/K+8Ig3J7LKYlNmvgJfeC4q40Lu6R4+IeGS7ukTGY4k5+jNvMbF/j4jYzS4yL28wsMS5uM7PEuLjNzBLj4jYzS4yL28wsMS5uM7PEuLjNzBLj4jYzS4yL28wsMS5uM7PEuLjNzBLj4jYzS4yL28wsMS5uM7PEuLjNzBLj4jYzS0zyNwuuNNFd9C21Rojq6vKOMGiFBDOndguwXVK75VrH+cvLXtZb3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmiXFxm5klxsVtZpYYF7eZWWJc3GZmifEdcBKwMdbxIKsIgqnMYLrm5h1pQClmvr9rORu6n6JW9RxXd2reccrSVmxl9Y6ldEY7ANNq53Bw3aE5pxpYCu+PnZueYdN3r6a7pRUBo191DE0nvJzOx9ey+crriI5OqsY3M+HsMyk01I9otr2uuCV9HPgIcHdEvCvvPC9URPAAKzmCRdQzihUsZkJMoVFNeUfrU4qZAaZUzWRa1Rzu67ot7yhlK1BgbsMxNFVNYGd0sbz1esZXT6GxqjnvaH1K5f2hqgLNbz+Z2ulTKbZ1sO78i2k4dDabL/8ZY888mfq5M2ldeictNy5l7OknjGi2vXGo5KPA64ajtCWN+B+6rWymgUZGqZGCCkxiGhtYM9IxBiXFzADNhUnUUJt3jEGpK4yiqWoCANWqYXRhLO3FHTmn6l8q74+qsU3UTp8KQKGhjpop+7NzSwtd6zZSN2cGAPWHzmbHXfePeLa9qrglfRuYCfyPpM9J+oGkFZJWSnpjtswfJB3a42dukbRA0ug+ln+fpBskLQEWj/Tv1EEb9TQ8O11PAx20jXSMQUkx896grbiNbd2bGFs9Me8o/Urx/bFzwxY6H1tD3axp1EydRNvdfwRgxx2r6d78zIjn2auKOyLOBtYArwZGA0si4phs+huSRgM/Ad4GIGkyMDki7gQ+18fyAEcCZ0TEK3t7XUlnSbpT0p1ddAzjb2jWu53RxartS5jbsJBqpfWpodIV2zvYcMmPaX7nqRQa6hn/gdNpXfIH1n7pYqK9A1VVjXimvW6Mu4cTgDdIOjebrgcOAn4K/Br4EqUCv2aA5QFujojNfb1QRFwGXAbQpHExlL9EHQ2099gaaaeNuh5bK5UoxcwpK0aRVTuWMLl2FpNqpucdZ0ApvT9iZzcbL7mK0cfOZ9SCeQDUTNmf/c/7IABd6zbQds8DI55rby5uAadHxPPWqqRNkg4HzgTO7m95SQuB7cMdti9NNNNGK22xnToaWM8TzOOYvOKUJcXMqYoI7m9bxujCGKbXzcs7TllSeX9EBJt+8DNqJk+k6aRFz87vbmmlqqmRKBbZesNvaXz1whHPtjcX903AxyR9LCJC0hERsTJ77CfAp4AxEXFvGcvnpqACc2I+K1lGEExhOo0ak3esfqWYGeDezlvZUlxPFx0sbb+WWdWHM7V6dt6x+vVM93rWdD1MY6GZ27b9HIBD6o9iYs20nJP1LZX3R8dDj7HjtpXUHHgAa79wEQBjzziBrvWbaF28HIBRR81j9KKjRjybIob0k33uJD0KLKC0lfwvwHGUxvIfiYhTs2UmAU8BX4mIf8jmNfS2vKT3AQsi4pxyXr9J42Kh/mZIfyfrnerq8o4waIUEM3e3tOQdYY88ePnIF+oLse78S+h45EmVs+xet8UdEdN7TH64j2XWs9vvHhFtvS0fEVcAVwxZQDOzF2ivOqrEzGxf4OI2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwS4+I2M0uMi9vMLDEubjOzxLi4zcwSs9fduixvkjYAjw3DU08ANg7D8w63FHM788hIMTMMX+6DI2JiOQu6uBMh6c6IWJB3jsFKMbczj4wUM0Nl5PZQiZlZYlzcZmaJcXGn47K8A+yhFHMPKrOksZI+OlxhyrTXr+cKkntuj3GbvUCSpgO/iIh5u82vjoiduYSyvVp13gHM9gIXALMkrQK6gHZgCzBX0gn0KHVJ5wKNEXG+pFnAvwETgR3A30XEn3P5DSwpLm6zF+4zwLyImC/pVcAvs+lHsq3xvlwGnB0RD0laCPw78JrhDmvpc3GbDb0VEfFIfwtIagSOA66WtGt23XAHs72Di9ts6G3v8f1OnnsQQH32bwF4JiLmj1gq22v4qBKzF24bsF8fj60H9pc0XlIdcCpARLQAj0h6K4BKXjoiaS153uI2e4EiYpOk30u6D2ijVNa7HuuS9GVgBfAU0HPn47uASyV9HqgB/gu4Z+SSW6p8OKCZWWI8VGJmlhgXt5lZYlzcZmaJcXGbmSXGxW1mlhgXt5lZYlzcZmaJcXGbmSXm/wOzXqtJYKqR9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_hat = model.predict(test_df.drop('label', axis=1))\n",
    "y_hat = y_hat.argmax(axis=1)\n",
    "y_hat = label_encoder.inverse_transform(y_hat)\n",
    "\n",
    "fig = plt.figure(figsize=(5,5))\n",
    "ax = plt.subplot()\n",
    "\n",
    "labels = list(test_df['label'].value_counts().index)\n",
    "\n",
    "confusion = confusion_matrix(test_label, y_hat, labels=labels)\n",
    "ax.matshow(confusion)\n",
    "\n",
    "ax.set_xticks(range(len(labels)))\n",
    "ax.set_yticks(range(len(labels)))\n",
    "\n",
    "ax.set_xticklabels(labels, rotation=90);\n",
    "ax.set_yticklabels(labels);\n",
    "\n",
    "for i in range(len(labels)):\n",
    "    for j in range(len(labels)):        \n",
    "        ax.text(j, i, confusion[i,j], va='center', ha='center')\n",
    "        \n",
    "plt.xlabel('true')\n",
    "plt.ylabel('predicted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:31.726281Z",
     "start_time": "2019-04-08T18:06:31.649100Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        roll       0.90      0.84      0.87        55\n",
      "  draw-cross       0.91      0.94      0.93        54\n",
      " draw-circle       0.92      0.97      0.95        37\n",
      "       floss       0.91      0.88      0.89        33\n",
      "       shake       0.85      0.88      0.87        33\n",
      "       fever       0.91      0.91      0.91        32\n",
      "\n",
      "   micro avg       0.90      0.90      0.90       244\n",
      "   macro avg       0.90      0.90      0.90       244\n",
      "weighted avg       0.90      0.90      0.90       244\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_label, y_hat, labels=labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-08T18:06:32.978127Z",
     "start_time": "2019-04-08T18:06:31.729890Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm: cannot remove 'models/v4/pb/1554746791': No such file or directory\n",
      "WARNING:tensorflow:From <ipython-input-27-618ac012a23b>:20: simple_save (from tensorflow.python.saved_model.simple_save) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.simple_save.\n",
      "WARNING:tensorflow:From /home/macermak/.local/share/virtualenvs/2019-demo4-ai-FZCiEg3v/lib/python3.6/site-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: models/v4/pb/1554746791/saved_model.pb\n",
      "\n",
      "Saved model:\n",
      "total 140\n",
      "-rw-r--r--. 1 macermak vpnusers 139114 Apr  8 20:06 saved_model.pb\n",
      "drwxr-xr-x. 2 macermak vpnusers   4096 Apr  8 20:06 variables\n",
      "\n",
      "Saved model:\n",
      "total 56\n",
      "-rw-r--r--. 1 macermak vpnusers 57240 Apr  8 20:06 saved_model.h5\n",
      "WARNING:tensorflow:This model was compiled with a Keras optimizer (<tensorflow.python.keras.optimizers.Adam object at 0x7fc733ba7a90>) but is being saved in TensorFlow format with `save_weights`. The model's weights will be saved, but unlike with TensorFlow optimizers in the TensorFlow format the optimizer's state will not be saved.\n",
      "\n",
      "Consider using a TensorFlow optimizer from `tf.train`.\n",
      "WARNING:tensorflow:From /home/macermak/.local/share/virtualenvs/2019-demo4-ai-FZCiEg3v/lib/python3.6/site-packages/tensorflow/python/keras/engine/network.py:1436: update_checkpoint_state (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.train.CheckpointManager to manage checkpoints rather than manually editing the Checkpoint proto.\n",
      "\n",
      "Saved model:\n",
      "total 28\n",
      "-rw-r--r--. 1 macermak vpnusers    77 Apr  8 20:06 checkpoint\n",
      "-rw-r--r--. 1 macermak vpnusers 14361 Apr  8 20:06 checkpoint.data-00000-of-00001\n",
      "-rw-r--r--. 1 macermak vpnusers   637 Apr  8 20:06 checkpoint.index\n",
      "-rw-r--r--. 1 macermak vpnusers  1236 Apr  8 20:06 saved_model_spec.json\n"
     ]
    }
   ],
   "source": [
    "save = True\n",
    "ver = int(dt.now().timestamp())\n",
    "\n",
    "if save:\n",
    "    for fmt in ['pb', 'hdf5', 'json']:\n",
    "        # save model as hdf5\n",
    "        MODEL_DIR = f\"models/v4/{fmt}\"\n",
    "        export_path = Path(MODEL_DIR, str(ver))\n",
    "\n",
    "        if fmt == 'pb':\n",
    "            # simple_save can not handle existing repository\n",
    "            !rm -r {export_path}\n",
    "\n",
    "            # Inspired by https://www.tensorflow.org/serving/tutorials/Serving_REST_simple\n",
    "            # to save model for TF-serving\n",
    "            tf.saved_model.simple_save(\n",
    "                tf.keras.backend.get_session(),\n",
    "                str(export_path),\n",
    "                inputs={'input_data': model.input},\n",
    "                outputs={t.name:t for t in model.outputs})\n",
    "            \n",
    "        elif fmt == 'hdf5':\n",
    "            # make sure directory exists\n",
    "            os.makedirs(export_path, exist_ok=True)\n",
    "            \n",
    "            model_path = Path(export_path, 'saved_model.h5') \n",
    "            \n",
    "            # hdf5\n",
    "            model.save(model_path)\n",
    "            \n",
    "        else:\n",
    "            # make sure directory exists\n",
    "            os.makedirs(export_path, exist_ok=True)\n",
    "            \n",
    "            # json\n",
    "            model_spec_path = Path(\n",
    "                export_path, 'saved_model_spec.json'\n",
    "            )\n",
    "            model_weights_path = Path(\n",
    "                export_path, 'checkpoint')\n",
    "            \n",
    "            model_spec_path.write_text(model.to_json(), encoding='utf-8')\n",
    "            model.save_weights(str(model_weights_path))\n",
    "\n",
    "        print('\\nSaved model:')\n",
    "        !ls -l {export_path}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Knative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "oc start-build tf-serving --from-dir ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "builds = !oc get builds\n",
    "latest = builds[-1].split()[0]\n",
    "\n",
    "!oc logs -f build.build.openshift.io/$latest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "revision = int(dt.timestamp(dt.now()))\n",
    "\n",
    "!oc process tf-serving-knative -p REVISION=$revision | oc apply -f -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "while oc get pods | grep tf-serving-knative | grep -v Running; do \n",
    "    oc get pods | grep tf-serving-knative\n",
    "    sleep 1\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash --out KSERVICE_DOMAIN\n",
    "\n",
    "oc get kservice tf-serving-knative-demo -o jsonpath='{.status.domain}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "KSERVICE_DOMAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "INGRESS_GATEWAY=\"istio-ingressgateway.istio-system.svc.cluster.local\"\n",
    "\n",
    "data = {\n",
    "  \"instances\": [\n",
    "    [\n",
    "      0.625899,\n",
    "      0.23741,\n",
    "      0.035971,\n",
    "      0.035971,\n",
    "      0.028777,\n",
    "      0.014388,\n",
    "      0.021583,\n",
    "      0,\n",
    "      0,\n",
    "      4.7488,\n",
    "      139,\n",
    "      1.359591,\n",
    "      1.667831\n",
    "    ]\n",
    "  ],\n",
    "  \"signature\": \"serving_default\"\n",
    "}\n",
    "\n",
    "json_response = requests.post(\"http://%s/model/predict\" % INGRESS_GATEWAY, json=data, headers={\"Host\": KSERVICE_DOMAIN, \"Content-Type\":\"application/json\"})\n",
    "json_response.content"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python TF 1.13",
   "language": "python",
   "name": "tf13"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}