notebooks/IntroDL_TSF.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "IntroDL_TSF.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/carranza96/DLSeminar/blob/master/notebooks/IntroDL_TSF.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mpPvqZB5ItB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%pip install tensorflow==2.1\n",
        "%pip install keras-tcn\n",
        "%matplotlib inline\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Dropout, Flatten, Conv1D, MaxPool1D, LSTM, Input, GRU\n",
        "from tensorflow.keras import backend as K\n",
        "from tcn import TCN\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from collections import defaultdict\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "from sklearn.metrics import mean_absolute_error\n",
        "import datetime\n",
        "import random\n",
        "import math"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WIXHuKXaTCBp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Fix random seed to allow reproducible experiments\n",
        "np.random.seed(1)\n",
        "tf.random.set_seed(1)\n",
        "random.seed(1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lncB6eMg5MXb",
        "colab_type": "text"
      },
      "source": [
        "# Load and inspect USA Temperature data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u5gM5f1crTN-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir data\n",
        "!wget -O data/test.csv https://www.dropbox.com/s/8f79c3upf65i1f7/test.csv?dl=1\n",
        "!wget -O data/train.csv https://www.dropbox.com/s/g2w93sfg0hkmjz0/train.csv?dl=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RAMSpIoj42vl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Read training set\n",
        "train_filename = 'data/train.csv'\n",
        "with open(train_filename, 'r') as datafile:\n",
        "  ts_list_train = datafile.readlines()\n",
        "  ts_list_train = np.asarray([np.asarray(l.rstrip().split(','), dtype=np.float32) for l in ts_list_train])\n",
        "\n",
        "# Read test set\n",
        "test_filename = 'data/test.csv'\n",
        "with open(test_filename, 'r') as datafile:\n",
        "  ts_list_test = datafile.readlines()\n",
        "  ts_list_test = np.asarray([np.asarray(l.rstrip().split(','), dtype=np.float32) for l in ts_list_test])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FXyvIHRd62tj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "headers = ['San Francisco', 'Los Angeles', 'Las Vegas', 'Dallas', 'Houston', 'Chicago', 'Detroit', 'Miami', 'Toronto', 'Philadelphia', 'New York', 'Boston']\n",
        "initial_moment = datetime.datetime(2012, 10, 1, 13,0)\n",
        "colors = plt.get_cmap('Set3', len(headers))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hALs9_0e7KCl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for i in range(len(headers)):\n",
        "  print(\"Length of {0} time serie: {1} hours. From {3} to {2}.\".format(headers[i], ts_list_train[i].shape[0], initial_moment + datetime.timedelta(hours=ts_list_train[i].shape[0]-1), initial_moment))\n",
        "\n",
        "print(\"\\nForecasting horizon: {0} hours.\".format(ts_list_test[0].shape[0]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1qE3PHB59_Lp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Lets see how the temperature have evolved over time\n",
        "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,6))\n",
        "ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%1.2f ºK'))\n",
        "\n",
        "for i in range(len(headers)):\n",
        "  x = [initial_moment + datetime.timedelta(hours=j, minutes=0) for j  in range(ts_list_train[i].shape[0])]\n",
        "  ax.plot(x, ts_list_train[i], label=headers[i], c=colors(i))\n",
        "\n",
        "ax.legend()\n",
        "ax.set_title('Temperature')\n",
        "plt.show()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNnz29QrAT_z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "fig, ax = plt.subplots(nrows=math.ceil(len(headers)/2), ncols=2, figsize=(15,15), sharex=True)\n",
        "\n",
        "for i in range(len(headers)):\n",
        "  x = [initial_moment + datetime.timedelta(hours=j, minutes=0) for j  in range(ts_list_train[i].shape[0])]\n",
        "  ax[i//2][i%2].plot(x, ts_list_train[i], label=headers[i], c=colors(i))\n",
        "  ax[i//2][i%2].set_title(headers[i])\n",
        "  ax[i//2][i%2].yaxis.set_major_formatter(ticker.FormatStrFormatter('%1.2f ºK'))\n",
        "  ax[i//2][i%2].xaxis.set_major_locator(plt.MaxNLocator(5))\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rW4-CDbhTRWB",
        "colab_type": "text"
      },
      "source": [
        "# Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dJv8BROdT4xK",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "It is important to scale features before training a neural network. Min-max normalization is a common way of doing this scaling.You could also use any other normalization method that rescales the values into a range of [0,1].\n",
        "\n",
        "> MIN-MAX NORMALIZATION <br/>\n",
        "><img src=\"https://www.oreilly.com/library/view/regression-analysis-with/9781788627306/assets/ffb3ac78-fd6f-4340-aa92-cde8ae0322d6.png\" alt=\"min max formula\" width=\"185\"/>\n",
        "><br/>\n",
        "><img src=\"https://www.researchgate.net/publication/282541174/figure/fig1/AS:307388692353061@1450298583749/Min-max-method-of-normalization.png\" alt=\"min max formula\" width=\"350\"/>\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qUJdDtddLemt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize(ts, norm_params):\n",
        "  \"\"\"\n",
        "  Apply min-max normalization\n",
        "  :param data: time series\n",
        "  :param norm_params: tuple with params mean, std, max, min\n",
        "  :return: normalized time series\n",
        "  \"\"\"\n",
        "  return (ts - norm_params['min']) / (norm_params['max'] - norm_params['min'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U6Gl8j2jUK3b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Let's normalize the data.\n",
        "ts_list_train_norm = []\n",
        "# Save training norm params in order to use it for the test data \n",
        "norm_params_list = []\n",
        "\n",
        "for ts in ts_list_train:\n",
        "  norm_params = {}\n",
        "  norm_params['mean'] = ts.mean()\n",
        "  norm_params['std'] = ts.std()\n",
        "  norm_params['max'] = ts.max()\n",
        "  norm_params['min'] = ts.min()\n",
        "  norm_params_list.append(norm_params)\n",
        "\n",
        "  ts_norm = normalize(ts, norm_params)\n",
        "  ts_list_train_norm.append(ts_norm)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "24iSdDmFYpk_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Lets normalize the test data\n",
        "ts_list_test_norm = []\n",
        "for i, ts in enumerate(ts_list_test):\n",
        "  norm_params = norm_params_list[i]\n",
        "  ts_norm = normalize(ts, norm_params)\n",
        "  ts_list_test_norm.append(ts_norm)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ZOmpIKzYwxQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# This is how the time series look like now. \n",
        "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,6))\n",
        "\n",
        "for i in range(len(headers)):\n",
        "  x = [initial_moment - datetime.timedelta(hours=i, minutes=0) for i  in range(ts_list_train_norm[i].shape[0])]\n",
        "  ax.plot(x, ts_list_train_norm[i], label=headers[i], c=colors(i))\n",
        "\n",
        "ax.legend()\n",
        "ax.set_title('Temperature (Normalized data)')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k_hDeIVWcqsF",
        "colab_type": "text"
      },
      "source": [
        "Let's now create the data for the model. The model will be given the last 240 hours (10 days) recorded temperature values, and needs to learn to predict the temperature for the next 24 hours (1 day). "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f10lnnrXercs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "past_history = 240\n",
        "forecast_horizon = 24"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kBq-EwAeD8w0",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "<img src=\"https://cdn-images-1.medium.com/max/1000/1*v5_QpzkQfufVogeCY9eaOw.png\" alt=\"RNN data input\" width=\"500\"/>\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ynYcd9r3e9cT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Moving window strategy\n",
        "\n",
        "x_train, y_train = [], []\n",
        "x_test, y_test = [], []\n",
        "\n",
        "for i, ts in enumerate(ts_list_train_norm):\n",
        "  # Train data\n",
        "  ts_x_train, ts_y_train = [], []\n",
        "  for j in range(0, ts.shape[0] - forecast_horizon + 1):\n",
        "      indices = range(j - past_history, j, 1)\n",
        "      # Reshape data from (past_history,) to (past_history, 1)\n",
        "      ts_x_train.append(np.reshape(ts[indices], (past_history, 1)))\n",
        "      ts_y_train.append(ts[j:j + forecast_horizon])\n",
        "  x_train.extend(np.asarray(ts_x_train))\n",
        "  y_train.extend(np.asarray(ts_y_train))\n",
        "  # Test data\n",
        "  ts_x_test = np.reshape(np.asarray(ts[-past_history:]), (past_history, 1))\n",
        "  ts_y_test=  ts_list_test_norm[i]\n",
        "  x_test.extend(np.asarray([ts_x_test]))\n",
        "  y_test.extend(np.asarray([ts_y_test]))\n",
        "  \n",
        "\n",
        "x_train, y_train = np.asarray(x_train), np.asarray(y_train)\n",
        "x_test, y_test = np.asarray(x_test), np.asarray(y_test, dtype='float32')\n",
        "\n",
        "print(\"TRAINING DATA\")\n",
        "print(\"Input shape\", x_train.shape)\n",
        "print(\"Output_shape\", y_train.shape)\n",
        "print()\n",
        "print(\"TEST DATA\")\n",
        "print(\"Input shape\", x_test.shape)\n",
        "print(\"Output_shape\", y_test.shape)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BTNOIhYdAhn8",
        "colab_type": "text"
      },
      "source": [
        "Visualize some examples"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AwPcKpdIOQDp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def show_plot(x, y, y_pred=None):\n",
        "  fig, ax = plt.subplots(1,1, figsize=(15,5))\n",
        "  ax.plot(x, 'o-', c=colors(3), markersize=3.5, label='History')\n",
        "  ax.plot([m for m in range(x.shape[0],x.shape[0]+y.shape[0])], y, 'x-', c=colors(0), markersize=3.5, label='True Future')\n",
        "  if y_pred is not None:\n",
        "    ax.plot([m for m in range(x.shape[0],x.shape[0]+y.shape[0])], y_pred, 'o-', c=colors(2), markersize=3.5, label='Model prediction')\n",
        "  ax.legend()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gYmZ5aYiAe0L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for _ in range(3):\n",
        "  random_index = random.randint(0,x_train.shape[0])\n",
        "  show_plot(x_train[random_index],y_train[random_index])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EF8FpScZHzga",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import defaultdict\n",
        "results = defaultdict(lambda: {})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZK61CFWKN1i",
        "colab_type": "text"
      },
      "source": [
        "# Long-Short Term Memory (LSTM) Neural Network "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IaYU2L5ULSCT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, LSTM, Input"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qbaxl-qnCJFU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create model with one LSTM layer with 64 units\n",
        "inp = Input(shape=x_train.shape[-2:])\n",
        "x = LSTM(64)(inp)\n",
        "x = Dense(forecast_horizon)(x)\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(optimizer='adam', loss='mae')\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gxMkOP0mNcKQ",
        "colab_type": "text"
      },
      "source": [
        "Lets train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2E1EYHcUNbgg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history = model.fit(x_train, y_train,\n",
        "          batch_size=256,\n",
        "          epochs=5,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mc20RivBVT-j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## Visualize training stats \n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WHY4o_RPV0X6",
        "colab_type": "text"
      },
      "source": [
        "Let's see the predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X8Wq8iz6Vuab",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predictions = model.predict(x_test)\n",
        "mae = mean_absolute_error(y_test, predictions)\n",
        "results['MAE']['SimpleLSTM'] = mae\n",
        "results['Y_PRED']['SimpleLSTM'] = predictions\n",
        "pd.DataFrame(results)['MAE']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QN3sv7DLV5TM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for x, y, y_pred in zip(x_test, y_test, predictions):\n",
        "  show_plot(x,y,y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vYxL3AZgXez3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can create a more-sofisticated model by stacking LSTM layers. \n",
        "inp = Input(shape=x_train.shape[-2:])\n",
        "x = LSTM(64, return_sequences=True)(inp)\n",
        "x = LSTM(32, return_sequences=False)(x)\n",
        "x = Dense(64)(x)\n",
        "x = Dense(forecast_horizon)(x)\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(optimizer='adam', loss='mae')\n",
        "print(model.summary())\n",
        "\n",
        "history = model.fit(x_train, y_train,\n",
        "          batch_size=256,\n",
        "          epochs=5,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "\n",
        "predictions = model.predict(x_test)\n",
        "mae = mean_absolute_error(y_test, predictions)\n",
        "results['MAE']['LSTM'] = mae\n",
        "results['Y_PRED']['LSTM'] = predictions\n",
        "print(pd.DataFrame(results)['MAE'])\n",
        "\n",
        "for x, y, y_pred in zip(x_test, y_test, predictions):\n",
        "  show_plot(x,y,y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TGrdjk8tG2EQ",
        "colab_type": "text"
      },
      "source": [
        " # Gated Recurrent Unit (GRU) Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_F02zm0nkVVn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# GRU\n",
        "inp = Input(shape=x_train.shape[-2:])\n",
        "x = GRU(128, return_sequences=True)(inp)\n",
        "x = GRU(64, return_sequences=True)(x)\n",
        "x = Flatten()(x)\n",
        "x = Dense(64)(x)\n",
        "x = Dense(forecast_horizon)(x)\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(optimizer='adam', loss='mae')\n",
        "print(model.summary())\n",
        "\n",
        "history = model.fit(x_train, y_train,\n",
        "          batch_size=256,\n",
        "          epochs=5,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "\n",
        "predictions = model.predict(x_test)\n",
        "mae = mean_absolute_error(y_test, predictions)\n",
        "results['MAE']['GRU'] = mae\n",
        "results['Y_PRED']['GRU'] = predictions\n",
        "print(pd.DataFrame(results)['MAE'])\n",
        "\n",
        "for x, y, y_pred in zip(x_test, y_test, predictions):\n",
        "  show_plot(x,y,y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1uJSRXwLG7nW",
        "colab_type": "text"
      },
      "source": [
        "# Temporal Convolutional Network (TCN)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PYMBPuVroA5F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# TCN\n",
        "inp = Input(shape=x_train.shape[-2:])\n",
        "x = TCN(nb_filters=64, kernel_size=3, nb_stacks=1, dilations=[1,2,4,8,16,32,64])(inp)\n",
        "x = Dense(forecast_horizon)(x)\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(optimizer='adam', loss='mae')\n",
        "print(model.summary())\n",
        "\n",
        "history = model.fit(x_train, y_train,\n",
        "          batch_size=256,\n",
        "          epochs=5,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "\n",
        "predictions = model.predict(x_test)\n",
        "mae = mean_absolute_error(y_test, predictions)\n",
        "results['MAE']['TCN'] = mae\n",
        "results['Y_PRED']['TCN'] = predictions\n",
        "print(pd.DataFrame(results)['MAE'])\n",
        "\n",
        "for x, y, y_pred in zip(x_test, y_test, predictions):\n",
        "  show_plot(x,y,y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q0myjLjkHC3K",
        "colab_type": "text"
      },
      "source": [
        "# Convolutional Neural Network (CNN)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6w2Z5684k1P_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# CNN\n",
        "inp = Input(shape=x_train.shape[-2:])\n",
        "x = Conv1D(128, 7, activation='relu', padding='same')(inp)\n",
        "x = MaxPool1D(pool_size=2)(x)\n",
        "x = tf.keras.layers.Conv1D(64, 5, activation='relu', padding='same')(inp)\n",
        "x = tf.keras.layers.MaxPool1D(pool_size=2)(x)\n",
        "x = tf.keras.layers.Flatten()(x)\n",
        "x = Dense(forecast_horizon)(x)\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(optimizer='adam', loss='mae')\n",
        "print(model.summary())\n",
        "\n",
        "history = model.fit(x_train, y_train,\n",
        "          batch_size=256,\n",
        "          epochs=5,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "\n",
        "predictions = model.predict(x_test)\n",
        "mae = mean_absolute_error(y_test, predictions)\n",
        "results['MAE']['CNN'] = mae\n",
        "results['Y_PRED']['CNN'] = predictions\n",
        "print(pd.DataFrame(results)['MAE'])\n",
        "\n",
        "for x, y, y_pred in zip(x_test, y_test, predictions):\n",
        "  show_plot(x,y,y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8L4QT1xzHKFY",
        "colab_type": "text"
      },
      "source": [
        "# Multi-Layer Perceptron (MLP)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gIEVQZGSmBnm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# MLP\n",
        "inp = Input(shape=x_train.shape[-2:])\n",
        "x = Flatten()(inp)\n",
        "x = Dense(128)(x)\n",
        "x = Dense(64)(x)\n",
        "x = Dense(32)(x)\n",
        "x = Dense(forecast_horizon)(x)\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(optimizer='adam', loss='mae')\n",
        "print(model.summary())\n",
        "\n",
        "history = model.fit(x_train, y_train,\n",
        "          batch_size=256,\n",
        "          epochs=5,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "\n",
        "predictions = model.predict(x_test)\n",
        "mae = mean_absolute_error(y_test, predictions)\n",
        "results['MAE']['MLP'] = mae\n",
        "results['Y_PRED']['MLP'] = predictions\n",
        "print(pd.DataFrame(results)['MAE'])\n",
        "\n",
        "for x, y, y_pred in zip(x_test, y_test, predictions):\n",
        "  show_plot(x,y,y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P15nj_RxHOPt",
        "colab_type": "text"
      },
      "source": [
        "# Analysis of results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cf3aWCFdPzZI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pd.DataFrame(results)['MAE']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m2mo6VALk9CZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for i, (x, y)  in enumerate(zip(x_test, y_test)):\n",
        "  y_pred=None\n",
        "  fig, ax = plt.subplots(1,1, figsize=(15,5))\n",
        "  ax.plot(x, 'o-', c=colors(3), markersize=3.5, label='History')\n",
        "  ax.plot([m for m in range(x.shape[0],x.shape[0]+y.shape[0])], y, 'x-', c=colors(0), markersize=3.5, label='True Future')\n",
        "  for j, (model_name, preds) in enumerate(results['Y_PRED'].items()): \n",
        "    y_pred = preds[i]\n",
        "    ax.plot([m for m in range(x.shape[0],x.shape[0]+y.shape[0])], y_pred, 'o-', c=colors(2+j), alpha=0.7, markersize=3.5, label=model_name)\n",
        "  ax.legend()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}