command_error_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data\n",
      "Command accuracy 0: 0.6923076923076923, out of 117\n",
      "Command accuracy 1: 0.0, out of 141\n",
      "Command accuracy 2: 0.0, out of 141\n",
      "Command accuracy 3: 0.6060606060606061, out of 132\n",
      "Command accuracy 4: 0.860655737704918, out of 122\n",
      "Command accuracy 5: 0.0, out of 124\n",
      "Command accuracy 6: 0.0, out of 99\n",
      "Command accuracy 7: 0.0, out of 97\n",
      "Command accuracy 8: 0.0, out of 135\n",
      "Command accuracy 9: 0.0, out of 92\n",
      "Overalla accuracy:  0.22166666666666668\n",
      "==================================================\n",
      "Validation Data\n",
      "Command accuracy 0: 0.8235294117647058, out of 34\n",
      "Command accuracy 1: 0.0, out of 32\n",
      "Command accuracy 2: 0.0, out of 36\n",
      "Command accuracy 3: 0.42424242424242425, out of 33\n",
      "Command accuracy 4: 0.8333333333333334, out of 30\n",
      "Command accuracy 5: 0.0, out of 33\n",
      "Command accuracy 6: 0.0, out of 25\n",
      "Command accuracy 7: 0.0, out of 25\n",
      "Command accuracy 8: 0.0, out of 32\n",
      "Command accuracy 9: 0.0, out of 20\n",
      "Overalla accuracy:  0.22333333333333333\n"
     ]
    }
   ],
   "source": [
    "from model.model import ImageCommandConverter\n",
    "from model.train_model import data_transforms\n",
    "import numpy as np\n",
    "from model.data_set import get_datasets, ImageDataSet\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "check_path = 'checkpoints/lightning_logs/version_4/checkpoints/epoch=33-step=271.ckpt'\n",
    "model = ImageCommandConverter.load_from_checkpoint(check_path).eval()\n",
    "\n",
    "\n",
    "train_set, val_set = get_datasets()\n",
    "print('Train Data')\n",
    "train_dataset = ImageDataSet(train_set, transform=data_transforms['validation'])\n",
    "train_loader = DataLoader(train_dataset, batch_size=100,\n",
    "                            shuffle=True,\n",
    "                            num_workers=0)\n",
    "res = {v: [0, 0] for v in range(10)}\n",
    "for item in train_loader:\n",
    "    x, t = item\n",
    "    t_h = model(x)\n",
    "    pred_command = np.argmax(t_h.detach().numpy()[:,:10], axis=1)\n",
    "    expected_command = t.detach().numpy()[:,0].astype(int)\n",
    "    vs, counts = np.unique(expected_command, return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][0] += c\n",
    "    vs, counts = np.unique(expected_command[np.where(pred_command == expected_command)], return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][1] += c\n",
    "for v in res:\n",
    "    print(f'Command accuracy {v}: {res[v][1]/res[v][0]}, out of {res[v][0]}')\n",
    "print('Overalla accuracy: ', sum([res[v][1] for v in res])/sum([res[v][0] for v in res]))\n",
    "print('='*50)\n",
    "print('Validation Data')\n",
    "val_dataset = ImageDataSet(val_set, transform=data_transforms['validation'])\n",
    "val_loader = DataLoader(val_dataset, batch_size=100,\n",
    "                            shuffle=True,\n",
    "                            num_workers=0)\n",
    "res = {v: [0, 0] for v in range(10)}\n",
    "for item in val_loader:\n",
    "    x, t = item\n",
    "    t_h = model(x)\n",
    "    pred_command = np.argmax(t_h.detach().numpy()[:,:10], axis=1)\n",
    "    expected_command = t.detach().numpy()[:,0].astype(int)\n",
    "    vs, counts = np.unique(expected_command, return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][0] += c\n",
    "    vs, counts = np.unique(expected_command[np.where(pred_command == expected_command)], return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][1] += c\n",
    "for v in res:\n",
    "    print(f'Command accuracy {v}: {res[v][1]/res[v][0]}, out of {res[v][0]}')\n",
    "print('Overalla accuracy: ', sum([res[v][1] for v in res])/sum([res[v][0] for v in res]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data\n",
      "Command accuracy 0: 0.6370370370370371, out of 135\n",
      "Command accuracy 1: 0.0, out of 123\n",
      "Command accuracy 2: 0.5294117647058824, out of 119\n",
      "Command accuracy 3: 0.18181818181818182, out of 154\n",
      "Command accuracy 4: 0.0, out of 127\n",
      "Command accuracy 5: 0.8739495798319328, out of 119\n",
      "Command accuracy 6: 0.0, out of 88\n",
      "Command accuracy 7: 0.0, out of 129\n",
      "Command accuracy 8: 0.0, out of 103\n",
      "Command accuracy 9: 0.0, out of 103\n",
      "Overalla accuracy:  0.23416666666666666\n",
      "==================================================\n",
      "Validation Data\n",
      "Command accuracy 0: 0.7647058823529411, out of 34\n",
      "Command accuracy 1: 0.0, out of 32\n",
      "Command accuracy 2: 0.37142857142857144, out of 35\n",
      "Command accuracy 3: 0.3235294117647059, out of 34\n",
      "Command accuracy 4: 0.0, out of 34\n",
      "Command accuracy 5: 0.8620689655172413, out of 29\n",
      "Command accuracy 6: 0.0, out of 14\n",
      "Command accuracy 7: 0.0, out of 25\n",
      "Command accuracy 8: 0.0, out of 32\n",
      "Command accuracy 9: 0.0, out of 31\n",
      "Overalla accuracy:  0.25\n"
     ]
    }
   ],
   "source": [
    "from model.model import ImageCommandConverter\n",
    "from model.train_model import data_transforms\n",
    "import numpy as np\n",
    "from model.data_set import get_datasets, ImageDataSet\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "check_path = 'checkpoints/lightning_logs/version_3/checkpoints/epoch=28-step=202.ckpt'\n",
    "model = ImageCommandConverter.load_from_checkpoint(check_path).eval()\n",
    "\n",
    "\n",
    "train_set, val_set = get_datasets()\n",
    "print('Train Data')\n",
    "train_dataset = ImageDataSet(train_set, transform=data_transforms['validation'])\n",
    "train_loader = DataLoader(train_dataset, batch_size=100,\n",
    "                            shuffle=True,\n",
    "                            num_workers=0)\n",
    "res = {v: [0, 0] for v in range(10)}\n",
    "for item in train_loader:\n",
    "    x, t = item\n",
    "    t_h = model(x)\n",
    "    pred_command = np.argmax(t_h.detach().numpy()[:,:10], axis=1)\n",
    "    expected_command = t.detach().numpy()[:,0].astype(int)\n",
    "    vs, counts = np.unique(expected_command, return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][0] += c\n",
    "    vs, counts = np.unique(expected_command[np.where(pred_command == expected_command)], return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][1] += c\n",
    "for v in res:\n",
    "    print(f'Command accuracy {v}: {res[v][1]/res[v][0]}, out of {res[v][0]}')\n",
    "print('Overalla accuracy: ', sum([res[v][1] for v in res])/sum([res[v][0] for v in res]))\n",
    "print('='*50)\n",
    "print('Validation Data')\n",
    "val_dataset = ImageDataSet(val_set, transform=data_transforms['validation'])\n",
    "val_loader = DataLoader(val_dataset, batch_size=100,\n",
    "                            shuffle=True,\n",
    "                            num_workers=0)\n",
    "res = {v: [0, 0] for v in range(10)}\n",
    "for item in val_loader:\n",
    "    x, t = item\n",
    "    t_h = model(x)\n",
    "    pred_command = np.argmax(t_h.detach().numpy()[:,:10], axis=1)\n",
    "    expected_command = t.detach().numpy()[:,0].astype(int)\n",
    "    vs, counts = np.unique(expected_command, return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][0] += c\n",
    "    vs, counts = np.unique(expected_command[np.where(pred_command == expected_command)], return_counts=True)\n",
    "    for v, c in zip(vs, counts):\n",
    "        res[v][1] += c\n",
    "for v in res:\n",
    "    print(f'Command accuracy {v}: {res[v][1]/res[v][0]}, out of {res[v][0]}')\n",
    "print('Overalla accuracy: ', sum([res[v][1] for v in res])/sum([res[v][0] for v in res]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}