exercise3_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Jun  8 15:06:34 2021       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 440.118.02   Driver Version: 440.118.02   CUDA Version: 10.2     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  GeForce RTX 208...  On   | 00000000:3D:00.0 Off |                  N/A |\n",
      "| 30%   28C    P8     6W / 250W |   2323MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  GeForce RTX 208...  On   | 00000000:3E:00.0 Off |                  N/A |\n",
      "| 29%   29C    P8     9W / 250W |     12MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  GeForce RTX 208...  On   | 00000000:3F:00.0 Off |                  N/A |\n",
      "| 37%   63C    P2   184W / 250W |   6437MiB / 11019MiB |     68%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  GeForce RTX 208...  On   | 00000000:40:00.0 Off |                  N/A |\n",
      "| 35%   56C    P2   100W / 250W |   6203MiB / 11019MiB |    100%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   4  GeForce RTX 208...  On   | 00000000:41:00.0 Off |                  N/A |\n",
      "| 36%   57C    P2   103W / 250W |   6203MiB / 11019MiB |    100%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   5  GeForce RTX 208...  On   | 00000000:60:00.0 Off |                  N/A |\n",
      "| 34%   55C    P2    98W / 250W |   6203MiB / 11019MiB |    100%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   6  GeForce RTX 208...  On   | 00000000:61:00.0 Off |                  N/A |\n",
      "| 37%   58C    P2   103W / 250W |   6203MiB / 11019MiB |    100%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   7  GeForce RTX 208...  On   | 00000000:62:00.0 Off |                  N/A |\n",
      "| 35%   56C    P2   108W / 250W |   6203MiB / 11019MiB |    100%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   8  GeForce RTX 208...  On   | 00000000:63:00.0 Off |                  N/A |\n",
      "| 34%   54C    P2   104W / 250W |   6203MiB / 11019MiB |    100%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   9  GeForce RTX 208...  On   | 00000000:64:00.0 Off |                  N/A |\n",
      "| 36%   58C    P2   107W / 250W |   6203MiB / 11019MiB |    100%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|    0     19976      C   python                                      2311MiB |\n",
      "|    2     38130      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6425MiB |\n",
      "|    3     38131      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6191MiB |\n",
      "|    4     38132      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6191MiB |\n",
      "|    5     38133      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6191MiB |\n",
      "|    6     38134      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6191MiB |\n",
      "|    7     38135      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6191MiB |\n",
      "|    8     38136      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6191MiB |\n",
      "|    9     38137      C   /home/kschen/ENTER/envs/yolov5/bin/python3  6191MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse, os\n",
    "import torch\n",
    "import random\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from utils.vdsr import Net\n",
    "from utils.dataset import DatasetFromHdf5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "training settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "batchSize = 128 # Training batch size\n",
    "nEpochs = 50 # Number of epochs to train for\n",
    "lr = 0.1 # Learning Rate\n",
    "step = 10 # Sets the learning rate to the initial LR decayed by momentum every n epochs\n",
    "resume = \"\" # Path to checkpoint\n",
    "start_epoch = 1 # Manual epoch number (useful on restarts)\n",
    "clip = 0.4 # Clipping Gradients\n",
    "momentum = 0.9\n",
    "weight_decay = 1e-4\n",
    "pretrained = '' # path to pretrained model\n",
    "checkpoint_step = 10 # the step to store checkpoints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GPU settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cuda = True # use cuda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Seed:  2910\n"
     ]
    }
   ],
   "source": [
    "seed = random.randint(1, 10000)\n",
    "print(\"Random Seed: \", seed)\n",
    "torch.manual_seed(seed)\n",
    "if cuda:\n",
    "    torch.cuda.manual_seed(seed)\n",
    "\n",
    "cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===> Loading datasets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jimmy15923/mnt/course/mediatek/MTKlab_SR/utils/dataset.py:8: H5pyDeprecationWarning: The default file mode will change to 'r' (read-only) in h5py 3.0. To suppress this warning, pass the mode you need to h5py.File(), or set the global default h5.get_config().default_file_mode, or set the environment variable H5PY_DEFAULT_READONLY=1. Available modes are: 'r', 'r+', 'w', 'w-'/'x', 'a'. See the docs for details.\n",
      "  hf = h5py.File(file_path)\n"
     ]
    }
   ],
   "source": [
    "print(\"===> Loading datasets\")\n",
    "train_set = DatasetFromHdf5(\"data/train.h5\")\n",
    "training_data_loader = DataLoader(dataset=train_set, num_workers=1, batch_size=batchSize, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===> Building model\n"
     ]
    }
   ],
   "source": [
    "print(\"===> Building model\")\n",
    "model = Net()\n",
    "criterion = nn.MSELoss(reduction='sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===> Setting GPU\n"
     ]
    }
   ],
   "source": [
    "print(\"===> Setting GPU\")\n",
    "if cuda:\n",
    "    model = model.cuda()\n",
    "    criterion = criterion.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optionally resume from a checkpoint\n",
    "if resume:\n",
    "    if os.path.isfile(resume):\n",
    "        print(\"=> loading checkpoint '{}'\".format(resume))\n",
    "        checkpoint = torch.load(resume)\n",
    "        start_epoch = checkpoint[\"epoch\"] + 1\n",
    "        model.load_state_dict(checkpoint[\"model\"].state_dict())\n",
    "    else:\n",
    "        print(\"=> no checkpoint found at '{}'\".format(resume))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optionally copy weights from a checkpoint\n",
    "if pretrained:\n",
    "    if os.path.isfile(pretrained):\n",
    "        print(\"=> loading model '{}'\".format(pretrained))\n",
    "        weights = torch.load(pretrained)\n",
    "        model.load_state_dict(weights['model'].state_dict())\n",
    "    else:\n",
    "        print(\"=> no model found at '{}'\".format(pretrained))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    \"\"\"Sets the learning rate to the initial LR decayed by 10 every 10 epochs\"\"\"\n",
    "    update_lr = lr * (0.5 ** (epoch // step))\n",
    "    return update_lr\n",
    "\n",
    "def train(training_data_loader, optimizer, model, criterion, epoch):\n",
    "    lr = adjust_learning_rate(optimizer, epoch-1)\n",
    "\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group[\"lr\"] = lr\n",
    "\n",
    "    print(\"Epoch = {}, lr = {}\".format(epoch, optimizer.param_groups[0][\"lr\"]))\n",
    "\n",
    "    model.train()\n",
    "\n",
    "    total_loss = 0\n",
    "    cnt = 0\n",
    "    for iteration, batch in enumerate(training_data_loader, 1):\n",
    "        input, target = Variable(batch[0]), Variable(batch[1], requires_grad=False)\n",
    "\n",
    "        if cuda:\n",
    "            input = input.cuda()\n",
    "            target = target.cuda()\n",
    "\n",
    "        loss = criterion(model(input), target)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward() \n",
    "        nn.utils.clip_grad_norm(model.parameters(), clip) \n",
    "        optimizer.step()\n",
    "        total_loss = total_loss + loss.data\n",
    "        cnt = cnt + 1\n",
    "\n",
    "    print(\"===> Epoch[{}]({}/{}): Loss: {:.10f}\".format(epoch, iteration, len(training_data_loader), total_loss/cnt))\n",
    "\n",
    "def save_checkpoint(model, epoch):\n",
    "    model_out_path = \"checkpoint/\" + \"model_epoch_{}.pth\".format(epoch)\n",
    "    state = {\"epoch\": epoch ,\"model\": model}\n",
    "    if not os.path.exists(\"checkpoint/\"):\n",
    "        os.makedirs(\"checkpoint/\")\n",
    "\n",
    "    torch.save(state, model_out_path)\n",
    "\n",
    "    print(\"Checkpoint saved to {}\".format(model_out_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===> Setting Optimizer\n",
      "===> Training\n",
      "Epoch = 1, lr = 0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jimmy15923/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:28: UserWarning: torch.nn.utils.clip_grad_norm is now deprecated in favor of torch.nn.utils.clip_grad_norm_.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===> Epoch[1](1134/1134): Loss: 114.3362426758\n",
      "Epoch = 2, lr = 0.1\n"
     ]
    }
   ],
   "source": [
    "print(\"===> Setting Optimizer\")\n",
    "optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)\n",
    "\n",
    "print(\"===> Training\")\n",
    "for epoch in range(start_epoch, nEpochs + 1):\n",
    "    train(training_data_loader, optimizer, model, criterion, epoch)\n",
    "    if epoch % checkpoint_step == 0:\n",
    "        save_checkpoint(model, epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Practice\n",
    "- What is the resize mode VDSR use in preparin the training data? (Nearnest, Biliner, or Bicubic?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}