L06_pytorch/code/adaline-with-autograd.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "STAT 479: Deep Learning (Spring 2019)  \n",
    "Instructor: Sebastian Raschka (sraschka@wisc.edu)  \n",
    "Course website: http://pages.stat.wisc.edu/~sraschka/teaching/stat479-ss2019/  \n",
    "GitHub repository: https://github.com/rasbt/stat479-deep-learning-ss19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The watermark extension is already loaded. To reload it, use:\n",
      "  %reload_ext watermark\n",
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.7.1\n",
      "IPython 7.2.0\n",
      "\n",
      "torch 1.0.1\n",
      "pandas 0.24.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch,pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ADALINE Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](images/adaline.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import grad\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('./datasets/iris.data', index_col=None, header=None)\n",
    "df.columns = ['x1', 'x2', 'x3', 'x4', 'y']\n",
    "df = df.iloc[50:150]\n",
    "df['y'] = df['y'].apply(lambda x: 0 if x == 'Iris-versicolor' else 1)\n",
    "\n",
    "\n",
    "# Assign features and target\n",
    "\n",
    "X = torch.tensor(df[['x2', 'x3']].values, dtype=torch.float)\n",
    "y = torch.tensor(df['y'].values, dtype=torch.int)\n",
    "\n",
    "# Shuffling & train/test split\n",
    "\n",
    "torch.manual_seed(123)\n",
    "shuffle_idx = torch.randperm(y.size(0), dtype=torch.long)\n",
    "\n",
    "X, y = X[shuffle_idx], y[shuffle_idx]\n",
    "\n",
    "percent70 = int(shuffle_idx.size(0)*0.7)\n",
    "\n",
    "X_train, X_test = X[shuffle_idx[:percent70]], X[shuffle_idx[percent70:]]\n",
    "y_train, y_test = y[shuffle_idx[:percent70]], y[shuffle_idx[percent70:]]\n",
    "\n",
    "# Normalize (mean zero, unit variance)\n",
    "\n",
    "mu, sigma = X_train.mean(dim=0), X_train.std(dim=0)\n",
    "X_train = (X_train - mu) / sigma\n",
    "X_test = (X_test - mu) / sigma"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ADALINE Manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Adaline1():\n",
    "    def __init__(self, num_features):\n",
    "        self.num_features = num_features\n",
    "        self.weight = torch.zeros(num_features, 1, \n",
    "                                  dtype=torch.float)\n",
    "        self.bias = torch.zeros(1, dtype=torch.float)\n",
    "\n",
    "    def forward(self, x):\n",
    "        netinputs = torch.add(torch.mm(x, self.weight), self.bias)\n",
    "        activations = netinputs\n",
    "        return activations.view(-1)\n",
    "        \n",
    "    def backward(self, x, yhat, y):  \n",
    "        \n",
    "        grad_loss_yhat = y - yhat\n",
    "        \n",
    "        grad_yhat_weight = x\n",
    "        grad_yhat_bias = 1.\n",
    "        \n",
    "        # Chain rule: inner times outer\n",
    "        grad_loss_weight = 2* -torch.mm(grad_yhat_weight.t(),\n",
    "                                      grad_loss_yhat.view(-1, 1)) / y.size(0)\n",
    "\n",
    "        grad_loss_bias = 2* -torch.sum(grad_yhat_bias*grad_loss_yhat) / y.size(0)\n",
    "        \n",
    "        return (-1)*grad_loss_weight, (-1)*grad_loss_bias\n",
    "    \n",
    "    \n",
    "    \n",
    "####################################################\n",
    "##### Training and evaluation wrappers\n",
    "###################################################\n",
    "\n",
    "def loss_func(yhat, y):\n",
    "    return torch.mean((yhat - y)**2)\n",
    "\n",
    "\n",
    "def train(model, x, y, num_epochs,\n",
    "          learning_rate=0.01, seed=123, minibatch_size=10):\n",
    "    cost = []\n",
    "    \n",
    "    torch.manual_seed(seed)\n",
    "    for e in range(num_epochs):\n",
    "        \n",
    "        #### Shuffle epoch\n",
    "        shuffle_idx = torch.randperm(y.size(0), dtype=torch.long)\n",
    "        minibatches = torch.split(shuffle_idx, minibatch_size)\n",
    "        \n",
    "        for minibatch_idx in minibatches:\n",
    "\n",
    "            #### Compute outputs ####\n",
    "            yhat = model.forward(x[minibatch_idx])\n",
    "\n",
    "            #### Compute gradients ####\n",
    "            negative_grad_w, negative_grad_b = \\\n",
    "                model.backward(x[minibatch_idx], yhat, y[minibatch_idx])\n",
    "\n",
    "            #### Update weights ####\n",
    "            model.weight += learning_rate * negative_grad_w\n",
    "            model.bias += learning_rate * negative_grad_b\n",
    "            \n",
    "            #### Logging ####\n",
    "            #minibatch_loss = loss(yhat, y[minibatch_idx])\n",
    "            #print('    Minibatch MSE: %.3f' % minibatch_loss)\n",
    "\n",
    "        #### Logging ####\n",
    "        yhat = model.forward(x)\n",
    "        curr_loss = loss_func(yhat, y)\n",
    "        print('Epoch: %03d' % (e+1), end=\"\")\n",
    "        print(' | MSE: %.5f' % curr_loss)\n",
    "        cost.append(curr_loss)\n",
    "\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001 | MSE: 0.38849\n",
      "Epoch: 002 | MSE: 0.31679\n",
      "Epoch: 003 | MSE: 0.26379\n",
      "Epoch: 004 | MSE: 0.22463\n",
      "Epoch: 005 | MSE: 0.19527\n",
      "Epoch: 006 | MSE: 0.17307\n",
      "Epoch: 007 | MSE: 0.15629\n",
      "Epoch: 008 | MSE: 0.14352\n",
      "Epoch: 009 | MSE: 0.13360\n",
      "Epoch: 010 | MSE: 0.12600\n",
      "Epoch: 011 | MSE: 0.12007\n",
      "Epoch: 012 | MSE: 0.11547\n",
      "Epoch: 013 | MSE: 0.11178\n",
      "Epoch: 014 | MSE: 0.10884\n",
      "Epoch: 015 | MSE: 0.10656\n",
      "Epoch: 016 | MSE: 0.10470\n",
      "Epoch: 017 | MSE: 0.10320\n",
      "Epoch: 018 | MSE: 0.10200\n",
      "Epoch: 019 | MSE: 0.10105\n",
      "Epoch: 020 | MSE: 0.10025\n"
     ]
    }
   ],
   "source": [
    "model = Adaline1(num_features=X_train.size(1))\n",
    "cost = train(model, \n",
    "             X_train, y_train.float(),\n",
    "             num_epochs=20,\n",
    "             learning_rate=0.01,\n",
    "             seed=123,\n",
    "             minibatch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 92.86\n",
      "Test Accuracy: 93.33\n",
      "Weights tensor([[-0.0189],\n",
      "        [ 0.3642]])\n",
      "Bias tensor([0.4580])\n"
     ]
    }
   ],
   "source": [
    "def custom_where(cond, x_1, x_2):\n",
    "    return (cond * x_1) + ((1-cond) * x_2)\n",
    "\n",
    "train_pred = model.forward(X_train)\n",
    "train_acc = torch.mean(\n",
    "    (custom_where(train_pred > 0.5, 1, 0).int() == y_train).float())\n",
    "\n",
    "test_pred = model.forward(X_test)\n",
    "test_acc = torch.mean(\n",
    "    (custom_where(test_pred > 0.5, 1, 0).int() == y_test).float())\n",
    "\n",
    "print('Training Accuracy: %.2f' % (train_acc*100))\n",
    "print('Test Accuracy: %.2f' % (test_acc*100))\n",
    "\n",
    "print('Weights', model.weight)\n",
    "print('Bias', model.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ADALINE Semi-Manually (using Autograd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Adaline2():\n",
    "    def __init__(self, num_features):\n",
    "        self.num_features = num_features\n",
    "        self.weight = torch.zeros(num_features, 1, \n",
    "                                  dtype=torch.float,\n",
    "                                  requires_grad=True)\n",
    "        self.bias = torch.zeros(1, dtype=torch.float,\n",
    "                                requires_grad=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        netinputs = torch.add(torch.mm(x, self.weight), self.bias)\n",
    "        activations = netinputs\n",
    "        return activations.view(-1)\n",
    "\n",
    "    \n",
    "####################################################\n",
    "##### Training and evaluation wrappers\n",
    "###################################################\n",
    "\n",
    "def loss_func(yhat, y):\n",
    "    return torch.mean((yhat - y)**2)\n",
    "\n",
    "\n",
    "def train(model, x, y, num_epochs,\n",
    "          learning_rate=0.01, seed=123, minibatch_size=10):\n",
    "    cost = []\n",
    "    \n",
    "    torch.manual_seed(seed)\n",
    "    for e in range(num_epochs):\n",
    "        \n",
    "        #### Shuffle epoch\n",
    "        shuffle_idx = torch.randperm(y.size(0), dtype=torch.long)\n",
    "        minibatches = torch.split(shuffle_idx, minibatch_size)\n",
    "        \n",
    "        for minibatch_idx in minibatches:\n",
    "\n",
    "            #### Compute outputs ####\n",
    "            yhat = model.forward(x[minibatch_idx])\n",
    "            loss = loss_func(yhat, y[minibatch_idx])\n",
    "            \n",
    "            #### Compute gradients ####\n",
    "            \n",
    "            negative_grad_w = grad(loss, model.weight, retain_graph=True)[0] * (-1)\n",
    "            negative_grad_b = grad(loss, model.bias)[0] * (-1)\n",
    "            \n",
    "            \n",
    "            #### Update weights ####\n",
    "            #model.weight += learning_rate * negative_grad_w\n",
    "            #model.bias += learning_rate * negative_grad_b\n",
    "\n",
    "            model.weight = model.weight + learning_rate * negative_grad_w\n",
    "            model.bias = model.bias + learning_rate * negative_grad_b\n",
    "\n",
    "        #### Logging ####\n",
    "        with torch.no_grad():\n",
    "            # context manager to\n",
    "            # avoid building graph during \"inference\"\n",
    "            # to save memory\n",
    "            yhat = model.forward(x)\n",
    "            curr_loss = loss_func(yhat, y)\n",
    "            print('Epoch: %03d' % (e+1), end=\"\")\n",
    "            print(' | MSE: %.5f' % curr_loss)\n",
    "            cost.append(curr_loss)\n",
    "\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001 | MSE: 0.38849\n",
      "Epoch: 002 | MSE: 0.31679\n",
      "Epoch: 003 | MSE: 0.26379\n",
      "Epoch: 004 | MSE: 0.22463\n",
      "Epoch: 005 | MSE: 0.19527\n",
      "Epoch: 006 | MSE: 0.17307\n",
      "Epoch: 007 | MSE: 0.15629\n",
      "Epoch: 008 | MSE: 0.14352\n",
      "Epoch: 009 | MSE: 0.13360\n",
      "Epoch: 010 | MSE: 0.12600\n",
      "Epoch: 011 | MSE: 0.12007\n",
      "Epoch: 012 | MSE: 0.11547\n",
      "Epoch: 013 | MSE: 0.11178\n",
      "Epoch: 014 | MSE: 0.10884\n",
      "Epoch: 015 | MSE: 0.10656\n",
      "Epoch: 016 | MSE: 0.10470\n",
      "Epoch: 017 | MSE: 0.10320\n",
      "Epoch: 018 | MSE: 0.10200\n",
      "Epoch: 019 | MSE: 0.10105\n",
      "Epoch: 020 | MSE: 0.10025\n"
     ]
    }
   ],
   "source": [
    "model = Adaline2(num_features=X_train.size(1))\n",
    "cost = train(model, \n",
    "             X_train, y_train.float(),\n",
    "             num_epochs=20,\n",
    "             learning_rate=0.01,\n",
    "             seed=123,\n",
    "             minibatch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 92.86\n",
      "Test Accuracy: 93.33\n",
      "Weights tensor([[-0.0189],\n",
      "        [ 0.3642]], grad_fn=<AddBackward0>)\n",
      "Bias tensor([0.4580], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "def custom_where(cond, x_1, x_2):\n",
    "    return (cond * x_1) + ((1-cond) * x_2)\n",
    "\n",
    "train_pred = model.forward(X_train)\n",
    "train_acc = torch.mean(\n",
    "    (custom_where(train_pred > 0.5, 1, 0).int() == y_train).float())\n",
    "\n",
    "test_pred = model.forward(X_test)\n",
    "test_acc = torch.mean(\n",
    "    (custom_where(test_pred > 0.5, 1, 0).int() == y_test).float())\n",
    "\n",
    "print('Training Accuracy: %.2f' % (train_acc*100))\n",
    "print('Test Accuracy: %.2f' % (test_acc*100))\n",
    "\n",
    "print('Weights', model.weight)\n",
    "print('Bias', model.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ADALINE Automatically"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Adaline3(torch.nn.Module):\n",
    "    def __init__(self, num_features):\n",
    "        super(Adaline3, self).__init__()\n",
    "        self.linear = torch.nn.Linear(num_features, 1)\n",
    "        \n",
    "        # change random weights to zero\n",
    "        # (don't do this for multi-layer nets!)\n",
    "        self.linear.weight.detach().zero_()\n",
    "        self.linear.bias.detach().zero_()\n",
    "\n",
    "    def forward(self, x):\n",
    "        netinputs = self.linear(x)\n",
    "        activations = netinputs\n",
    "        return activations.view(-1)\n",
    "\n",
    "    \n",
    "####################################################\n",
    "##### Training and evaluation wrappers\n",
    "###################################################\n",
    "\n",
    "\n",
    "def train(model, x, y, num_epochs,\n",
    "          learning_rate=0.01, seed=123, minibatch_size=10):\n",
    "    cost = []\n",
    "    \n",
    "    torch.manual_seed(seed)\n",
    "    \n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "    \n",
    "    for e in range(num_epochs):\n",
    "        \n",
    "        #### Shuffle epoch\n",
    "        shuffle_idx = torch.randperm(y.size(0), dtype=torch.long)\n",
    "        minibatches = torch.split(shuffle_idx, minibatch_size)\n",
    "        \n",
    "        for minibatch_idx in minibatches:\n",
    "\n",
    "            #### Compute outputs ####\n",
    "            yhat = model.forward(x[minibatch_idx])\n",
    "            \n",
    "            # you could also use our \"manual\" loss_func\n",
    "            loss = F.mse_loss(yhat, y[minibatch_idx])\n",
    "            \n",
    "            #### Reset gradients from previous iteration ####\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            #### Compute gradients ####\n",
    "            loss.backward()\n",
    "            \n",
    "            #### Update weights ####\n",
    "            optimizer.step()\n",
    "\n",
    "        #### Logging ####\n",
    "        with torch.no_grad():\n",
    "            # context manager to\n",
    "            # avoid building graph during \"inference\"\n",
    "            # to save memory\n",
    "            yhat = model.forward(x)\n",
    "            curr_loss = loss_func(yhat, y)\n",
    "            print('Epoch: %03d' % (e+1), end=\"\")\n",
    "            print(' | MSE: %.5f' % curr_loss)\n",
    "            cost.append(curr_loss)\n",
    "\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001 | MSE: 0.38849\n",
      "Epoch: 002 | MSE: 0.31679\n",
      "Epoch: 003 | MSE: 0.26379\n",
      "Epoch: 004 | MSE: 0.22463\n",
      "Epoch: 005 | MSE: 0.19527\n",
      "Epoch: 006 | MSE: 0.17307\n",
      "Epoch: 007 | MSE: 0.15629\n",
      "Epoch: 008 | MSE: 0.14352\n",
      "Epoch: 009 | MSE: 0.13360\n",
      "Epoch: 010 | MSE: 0.12600\n",
      "Epoch: 011 | MSE: 0.12007\n",
      "Epoch: 012 | MSE: 0.11547\n",
      "Epoch: 013 | MSE: 0.11178\n",
      "Epoch: 014 | MSE: 0.10884\n",
      "Epoch: 015 | MSE: 0.10656\n",
      "Epoch: 016 | MSE: 0.10470\n",
      "Epoch: 017 | MSE: 0.10320\n",
      "Epoch: 018 | MSE: 0.10200\n",
      "Epoch: 019 | MSE: 0.10105\n",
      "Epoch: 020 | MSE: 0.10025\n"
     ]
    }
   ],
   "source": [
    "model = Adaline3(num_features=X_train.size(1))\n",
    "cost = train(model, \n",
    "             X_train, y_train.float(),\n",
    "             num_epochs=20,\n",
    "             learning_rate=0.01,\n",
    "             seed=123,\n",
    "             minibatch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 92.86\n",
      "Test Accuracy: 93.33\n",
      "Weights Parameter containing:\n",
      "tensor([[-0.0189,  0.3642]], requires_grad=True)\n",
      "Bias Parameter containing:\n",
      "tensor([0.4580], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "def custom_where(cond, x_1, x_2):\n",
    "    return (cond * x_1) + ((1-cond) * x_2)\n",
    "\n",
    "train_pred = model.forward(X_train)\n",
    "train_acc = torch.mean(\n",
    "    (custom_where(train_pred > 0.5, 1, 0).int() == y_train).float())\n",
    "\n",
    "test_pred = model.forward(X_test)\n",
    "test_acc = torch.mean(\n",
    "    (custom_where(test_pred > 0.5, 1, 0).int() == y_test).float())\n",
    "\n",
    "print('Training Accuracy: %.2f' % (train_acc*100))\n",
    "print('Test Accuracy: %.2f' % (test_acc*100))\n",
    "\n",
    "print('Weights', model.linear.weight)\n",
    "print('Bias', model.linear.bias)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc-showcode": false,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 2
}