ViFi-CLIP_Inference_custom_video.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "eff034b9-55b2-4d3b-b4a7-3a9c2f7225aa",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Code for performing inference with ViFi-CLIP on custom videos\n",
    "\n",
    "### Please set the corresponding values in the cell below. Afterwards, just run the cells for inference with ViFi-CLIP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b5b1a2f2-a091-4efd-9477-0d158e5b4b3b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Set values here ###\n",
    "config = 'configs/zero_shot/train/k400/16_16_vifi_clip.yaml'\n",
    "output_folder_name = \"exp\"\n",
    "pretrained_model_path = \"/home/gamerx/Downloads/vifi_clip_10_epochs_k400_full_finetuned.pth\"\n",
    "# List the action names for which ViFi-CLIP will perform action recognition\n",
    "class_names = ['dancing', 'drum beating', 'swimming', \"climbing stairs\"]\n",
    "# Load your video example:\n",
    "video_path = '/home/gamerx/Downloads/hmdb51_org/climb_stairs/Walking_up_the_stairs_climb_stairs_l_cm_np1_ba_med_1.avi'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae8a87b3-c1e8-4654-8762-ffd033544482",
   "metadata": {},
   "source": [
    "### Import libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9b8d8a03-2bc5-4468-b682-eeb9f0fbcb8b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gamerx/anaconda3/envs/vclip/lib/python3.7/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "/home/gamerx/Documents/PhD_research/clean_repos/ViFi-CLIP/clip/clip.py:23: UserWarning: PyTorch version 1.7.1 or higher is recommended\n",
      "  warnings.warn(\"PyTorch version 1.7.1 or higher is recommended\")\n",
      "/home/gamerx/anaconda3/envs/vclip/lib/python3.7/site-packages/mmcv/__init__.py:21: UserWarning: On January 1, 2023, MMCV will release v2.0.0, in which it will remove components related to the training process and add a data transformation module. In addition, it will rename the package names mmcv to mmcv-lite and mmcv-full to mmcv. See https://github.com/open-mmlab/mmcv/blob/master/docs/en/compatibility.md for more details.\n",
      "  'On January 1, 2023, MMCV will release v2.0.0, in which it will remove '\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from utils.config import get_config\n",
    "from utils.logger import create_logger\n",
    "import time\n",
    "import numpy as np\n",
    "from utils.config import get_config\n",
    "from trainers import vificlip\n",
    "from datasets.pipeline import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb3879a8-2450-4863-8b6a-e7f48501771f",
   "metadata": {},
   "source": [
    "### Setting up configuration, no need to change anything."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "78418d43-ddd6-460d-8c30-7884a6383c1c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=> merge config from configs/zero_shot/train/k400/16_16_vifi_clip.yaml\n",
      "[2023-11-24 23:42:45 ViT-B/16](588939814.py 19): INFO working dir: exp\n"
     ]
    }
   ],
   "source": [
    "# Step 1:\n",
    "# Configuration class \n",
    "class parse_option():\n",
    "    def __init__(self):\n",
    "        self.config = config\n",
    "        self.output =  output_folder_name   # Name of output folder to store logs and save weights\n",
    "        self.resume = pretrained_model_path\n",
    "        # No need to change below args.\n",
    "        self.only_test = True\n",
    "        self.opts = None\n",
    "        self.batch_size = None\n",
    "        self.pretrained = None\n",
    "        self.accumulation_steps = None\n",
    "        self.local_rank = 0\n",
    "args = parse_option()\n",
    "config = get_config(args)\n",
    "# logger\n",
    "logger = create_logger(output_dir=args.output, name=f\"{config.MODEL.ARCH}\")\n",
    "logger.info(f\"working dir: {config.OUTPUT}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fda20686-733a-463b-8dbb-1b4bd1cb1eda",
   "metadata": {},
   "source": [
    "### Loading ViFi-CLIP and its pretrained weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9f7d8db5-35c7-4c42-82bb-2c58257f61a3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-11-24 23:42:45 ViT-B/16](vificlip.py 202): INFO Loading CLIP (backbone: ViT-B/16)\n",
      "[2023-11-24 23:42:49 ViT-B/16](vificlip.py 205): INFO Building ViFi-CLIP CLIP\n",
      "[2023-11-24 23:42:49 ViT-B/16](vificlip.py 222): INFO Turning on gradients for COMPLETE ViFi-CLIP model\n",
      "[2023-11-24 23:42:49 ViT-B/16](vificlip.py 245): INFO Parameters to be updated: {'text_encoder.transformer.resblocks.1.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.1.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.1.attn.out_proj.weight', 'text_encoder.transformer.resblocks.4.attn.in_proj_weight', 'image_encoder.transformer.resblocks.2.attn.in_proj_bias', 'image_encoder.transformer.resblocks.3.ln_1.weight', 'text_encoder.transformer.resblocks.11.attn.in_proj_bias', 'image_encoder.transformer.resblocks.3.attn.in_proj_bias', 'image_encoder.transformer.resblocks.10.ln_1.weight', 'image_encoder.transformer.resblocks.6.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.7.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.4.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.3.ln_1.bias', 'text_encoder.transformer.resblocks.2.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.9.ln_2.weight', 'image_encoder.transformer.resblocks.10.attn.in_proj_bias', 'image_encoder.conv1.weight', 'image_encoder.ln_post.bias', 'text_encoder.positional_embedding', 'image_encoder.transformer.resblocks.2.attn.out_proj.weight', 'text_encoder.transformer.resblocks.1.ln_1.weight', 'text_encoder.transformer.resblocks.8.ln_2.weight', 'image_encoder.transformer.resblocks.8.ln_2.weight', 'text_encoder.transformer.resblocks.7.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.7.attn.out_proj.weight', 'image_encoder.transformer.resblocks.8.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.1.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.11.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.1.attn.in_proj_bias', 'image_encoder.proj', 'image_encoder.transformer.resblocks.6.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.8.attn.out_proj.weight', 'text_encoder.transformer.resblocks.5.ln_2.weight', 'image_encoder.transformer.resblocks.1.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.8.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.7.attn.in_proj_bias', 'text_encoder.transformer.resblocks.7.attn.in_proj_weight', 'text_encoder.transformer.resblocks.11.attn.in_proj_weight', 'text_encoder.transformer.resblocks.11.ln_1.weight', 'text_encoder.transformer.resblocks.11.attn.out_proj.weight', 'text_encoder.transformer.resblocks.11.attn.out_proj.bias', 'image_encoder.transformer.resblocks.3.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.8.ln_1.weight', 'text_encoder.transformer.resblocks.2.ln_2.weight', 'image_encoder.transformer.resblocks.4.ln_1.weight', 'text_encoder.transformer.resblocks.1.ln_2.weight', 'text_encoder.transformer.resblocks.5.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.9.attn.out_proj.bias', 'image_encoder.transformer.resblocks.5.ln_1.weight', 'image_encoder.transformer.resblocks.0.ln_1.weight', 'image_encoder.transformer.resblocks.11.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.0.attn.in_proj_bias', 'text_encoder.transformer.resblocks.7.attn.out_proj.weight', 'text_encoder.transformer.resblocks.6.ln_1.bias', 'image_encoder.transformer.resblocks.9.ln_2.weight', 'image_encoder.transformer.resblocks.0.attn.in_proj_weight', 'image_encoder.transformer.resblocks.1.ln_2.weight', 'image_encoder.transformer.resblocks.9.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.9.attn.out_proj.weight', 'text_encoder.transformer.resblocks.10.mlp.c_proj.weight', 'image_encoder.ln_pre.bias', 'text_encoder.transformer.resblocks.8.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.10.attn.out_proj.bias', 'image_encoder.transformer.resblocks.9.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.8.attn.in_proj_weight', 'image_encoder.transformer.resblocks.6.ln_2.bias', 'text_encoder.transformer.resblocks.5.attn.out_proj.weight', 'text_encoder.transformer.resblocks.10.attn.out_proj.weight', 'text_encoder.transformer.resblocks.0.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.9.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.5.attn.out_proj.weight', 'image_encoder.transformer.resblocks.1.attn.out_proj.weight', 'image_encoder.transformer.resblocks.2.attn.out_proj.bias', 'text_encoder.transformer.resblocks.0.ln_2.weight', 'text_encoder.transformer.resblocks.6.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.0.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.5.attn.in_proj_weight', 'text_encoder.transformer.resblocks.10.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.9.mlp.c_fc.weight', 'image_encoder.class_embedding', 'text_encoder.transformer.resblocks.9.attn.in_proj_weight', 'text_encoder.transformer.resblocks.5.attn.in_proj_weight', 'image_encoder.transformer.resblocks.8.ln_1.bias', 'image_encoder.transformer.resblocks.9.ln_2.bias', 'text_encoder.transformer.resblocks.6.mlp.c_proj.bias', 'text_encoder.transformer.resblocks.7.ln_2.bias', 'text_encoder.transformer.resblocks.6.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.0.ln_1.bias', 'image_encoder.transformer.resblocks.3.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.3.ln_1.bias', 'image_encoder.transformer.resblocks.8.ln_2.bias', 'image_encoder.ln_post.weight', 'image_encoder.transformer.resblocks.5.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.4.attn.in_proj_bias', 'image_encoder.transformer.resblocks.10.ln_2.bias', 'image_encoder.transformer.resblocks.7.ln_2.weight', 'image_encoder.transformer.resblocks.10.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.2.attn.out_proj.weight', 'image_encoder.transformer.resblocks.0.ln_2.bias', 'image_encoder.transformer.resblocks.0.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.1.attn.in_proj_weight', 'image_encoder.transformer.resblocks.11.attn.in_proj_bias', 'image_encoder.transformer.resblocks.3.ln_2.weight', 'text_encoder.transformer.resblocks.3.ln_1.weight', 'text_encoder.transformer.resblocks.8.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.4.ln_2.weight', 'text_encoder.transformer.resblocks.1.ln_2.bias', 'image_encoder.transformer.resblocks.4.ln_1.bias', 'image_encoder.transformer.resblocks.7.ln_1.bias', 'image_encoder.transformer.resblocks.10.ln_1.bias', 'image_encoder.transformer.resblocks.0.attn.out_proj.weight', 'image_encoder.transformer.resblocks.3.attn.out_proj.bias', 'image_encoder.transformer.resblocks.9.ln_1.bias', 'image_encoder.transformer.resblocks.10.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.11.ln_2.weight', 'text_encoder.transformer.resblocks.2.attn.in_proj_weight', 'text_encoder.transformer.resblocks.3.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.5.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.7.ln_1.weight', 'text_encoder.transformer.resblocks.7.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.10.ln_2.weight', 'image_encoder.transformer.resblocks.11.attn.in_proj_weight', 'text_encoder.transformer.resblocks.2.ln_1.bias', 'image_encoder.transformer.resblocks.6.ln_2.weight', 'image_encoder.transformer.resblocks.7.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.7.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.6.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.10.mlp.c_proj.bias', 'text_encoder.transformer.resblocks.8.attn.in_proj_bias', 'image_encoder.transformer.resblocks.2.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.5.attn.out_proj.bias', 'text_encoder.transformer.resblocks.4.attn.out_proj.bias', 'image_encoder.transformer.resblocks.4.attn.out_proj.bias', 'image_encoder.transformer.resblocks.4.ln_2.bias', 'image_encoder.transformer.resblocks.8.attn.in_proj_weight', 'text_encoder.transformer.resblocks.4.attn.in_proj_bias', 'text_encoder.transformer.resblocks.2.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.8.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.10.mlp.c_fc.bias', 'text_encoder.ln_final.bias', 'text_encoder.transformer.resblocks.9.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.4.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.2.ln_2.bias', 'text_encoder.transformer.resblocks.11.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.11.attn.out_proj.weight', 'text_encoder.transformer.resblocks.1.attn.out_proj.bias', 'text_encoder.transformer.resblocks.5.ln_1.bias', 'image_encoder.transformer.resblocks.1.attn.in_proj_weight', 'image_encoder.transformer.resblocks.10.attn.out_proj.bias', 'image_encoder.transformer.resblocks.3.ln_2.bias', 'image_encoder.transformer.resblocks.6.attn.in_proj_weight', 'image_encoder.transformer.resblocks.4.attn.in_proj_weight', 'image_encoder.transformer.resblocks.6.ln_1.weight', 'text_encoder.transformer.resblocks.0.ln_1.weight', 'text_encoder.transformer.resblocks.1.ln_1.bias', 'text_encoder.transformer.resblocks.4.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.11.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.6.attn.out_proj.bias', 'image_encoder.transformer.resblocks.6.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.9.attn.out_proj.bias', 'text_encoder.transformer.resblocks.6.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.2.mlp.c_proj.bias', 'text_encoder.transformer.resblocks.5.attn.in_proj_bias', 'text_encoder.transformer.resblocks.10.attn.in_proj_weight', 'image_encoder.transformer.resblocks.11.ln_1.bias', 'text_encoder.transformer.resblocks.5.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.5.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.2.attn.in_proj_weight', 'image_encoder.transformer.resblocks.6.ln_1.bias', 'text_encoder.transformer.resblocks.4.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.1.attn.in_proj_bias', 'image_encoder.transformer.resblocks.0.attn.out_proj.bias', 'image_encoder.transformer.resblocks.5.ln_1.bias', 'image_encoder.transformer.resblocks.5.ln_2.bias', 'logit_scale', 'image_encoder.transformer.resblocks.9.attn.in_proj_weight', 'text_encoder.transformer.resblocks.9.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.3.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.0.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.8.attn.out_proj.bias', 'text_encoder.transformer.resblocks.8.attn.out_proj.weight', 'image_encoder.transformer.resblocks.8.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.2.ln_1.weight', 'text_encoder.transformer.resblocks.10.ln_1.bias', 'image_encoder.transformer.resblocks.7.ln_1.weight', 'image_encoder.transformer.resblocks.9.attn.in_proj_bias', 'text_encoder.transformer.resblocks.3.attn.in_proj_weight', 'text_encoder.transformer.resblocks.9.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.6.attn.out_proj.bias', 'image_encoder.transformer.resblocks.7.mlp.c_proj.bias', 'text_encoder.transformer.resblocks.3.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.4.ln_1.weight', 'text_encoder.transformer.resblocks.10.ln_2.bias', 'text_encoder.transformer.resblocks.3.ln_2.weight', 'image_encoder.transformer.resblocks.11.ln_2.bias', 'text_encoder.transformer.resblocks.2.ln_2.bias', 'image_encoder.transformer.resblocks.7.attn.in_proj_weight', 'text_encoder.transformer.resblocks.0.attn.out_proj.weight', 'text_encoder.transformer.resblocks.5.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.3.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.5.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.11.mlp.c_proj.bias', 'text_encoder.transformer.resblocks.5.ln_1.weight', 'text_encoder.transformer.resblocks.9.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.3.ln_2.bias', 'text_encoder.transformer.resblocks.5.attn.out_proj.bias', 'text_encoder.transformer.resblocks.0.attn.out_proj.bias', 'text_encoder.transformer.resblocks.8.ln_2.bias', 'text_encoder.transformer.resblocks.0.ln_1.bias', 'image_encoder.ln_pre.weight', 'image_encoder.transformer.resblocks.2.ln_2.weight', 'image_encoder.transformer.resblocks.0.attn.in_proj_bias', 'text_encoder.transformer.resblocks.2.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.2.attn.out_proj.bias', 'text_encoder.transformer.resblocks.3.attn.out_proj.bias', 'text_encoder.transformer.resblocks.6.ln_2.bias', 'image_encoder.transformer.resblocks.4.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.5.ln_2.weight', 'image_encoder.transformer.resblocks.0.ln_2.weight', 'image_encoder.transformer.resblocks.6.attn.out_proj.weight', 'image_encoder.transformer.resblocks.10.ln_2.weight', 'text_encoder.transformer.resblocks.4.ln_2.weight', 'text_encoder.transformer.resblocks.0.mlp.c_proj.bias', 'text_encoder.transformer.resblocks.1.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.6.attn.in_proj_weight', 'text_encoder.transformer.resblocks.11.ln_1.bias', 'text_encoder.transformer.resblocks.11.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.11.ln_2.weight', 'image_encoder.transformer.resblocks.2.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.2.ln_1.bias', 'image_encoder.positional_embedding', 'image_encoder.transformer.resblocks.3.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.7.attn.out_proj.bias', 'image_encoder.transformer.resblocks.5.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.8.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.10.attn.in_proj_weight', 'image_encoder.transformer.resblocks.2.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.2.ln_1.weight', 'text_encoder.transformer.resblocks.4.attn.out_proj.weight', 'image_encoder.transformer.resblocks.5.attn.in_proj_bias', 'text_encoder.transformer.resblocks.4.ln_1.bias', 'text_encoder.transformer.resblocks.8.ln_1.weight', 'text_encoder.transformer.resblocks.9.ln_1.weight', 'image_encoder.transformer.resblocks.11.attn.out_proj.bias', 'text_encoder.transformer.resblocks.11.ln_2.bias', 'text_encoder.transformer.resblocks.2.attn.in_proj_bias', 'image_encoder.transformer.resblocks.1.ln_1.bias', 'text_encoder.transformer.resblocks.4.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.4.attn.out_proj.weight', 'text_encoder.transformer.resblocks.6.attn.in_proj_bias', 'text_encoder.transformer.resblocks.7.attn.in_proj_bias', 'image_encoder.transformer.resblocks.9.ln_1.weight', 'text_encoder.transformer.resblocks.4.ln_2.bias', 'text_encoder.transformer.resblocks.8.attn.out_proj.bias', 'text_encoder.transformer.resblocks.9.attn.in_proj_bias', 'text_encoder.transformer.resblocks.0.ln_2.bias', 'text_encoder.transformer.resblocks.1.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.7.attn.out_proj.bias', 'text_encoder.transformer.resblocks.9.ln_2.bias', 'text_encoder.transformer.resblocks.3.attn.in_proj_bias', 'text_encoder.transformer.resblocks.11.mlp.c_proj.bias', 'text_encoder.transformer.resblocks.0.attn.in_proj_weight', 'image_encoder.transformer.resblocks.2.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.3.attn.out_proj.weight', 'text_encoder.text_projection', 'text_encoder.transformer.resblocks.6.attn.out_proj.weight', 'text_encoder.transformer.resblocks.7.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.6.ln_1.weight', 'image_encoder.transformer.resblocks.3.attn.in_proj_weight', 'image_encoder.transformer.resblocks.4.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.10.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.7.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.9.attn.out_proj.weight', 'image_encoder.transformer.resblocks.1.ln_1.weight', 'text_encoder.transformer.resblocks.3.mlp.c_proj.bias', 'image_encoder.transformer.resblocks.4.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.6.ln_2.weight', 'image_encoder.transformer.resblocks.6.attn.in_proj_bias', 'text_encoder.transformer.resblocks.11.mlp.c_fc.bias', 'text_encoder.transformer.resblocks.0.mlp.c_proj.weight', 'image_encoder.transformer.resblocks.1.attn.out_proj.bias', 'text_encoder.transformer.resblocks.7.ln_2.weight', 'image_encoder.transformer.resblocks.10.attn.out_proj.weight', 'image_encoder.transformer.resblocks.11.ln_1.weight', 'image_encoder.transformer.resblocks.0.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.1.ln_2.bias', 'text_encoder.transformer.resblocks.5.ln_2.bias', 'text_encoder.transformer.resblocks.0.mlp.c_fc.weight', 'image_encoder.transformer.resblocks.1.mlp.c_fc.weight', 'text_encoder.transformer.resblocks.1.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.7.ln_1.bias', 'text_encoder.transformer.resblocks.10.ln_1.weight', 'text_encoder.transformer.resblocks.8.ln_1.bias', 'text_encoder.ln_final.weight', 'image_encoder.transformer.resblocks.8.attn.in_proj_bias', 'image_encoder.transformer.resblocks.10.mlp.c_fc.bias', 'image_encoder.transformer.resblocks.8.mlp.c_proj.weight', 'text_encoder.transformer.resblocks.10.attn.in_proj_bias', 'image_encoder.transformer.resblocks.3.attn.out_proj.weight', 'text_encoder.transformer.resblocks.9.ln_1.bias', 'image_encoder.transformer.resblocks.7.ln_2.bias'}\n",
      "[2023-11-24 23:42:49 ViT-B/16](vificlip.py 246): INFO Total learnable items: 301\n"
     ]
    }
   ],
   "source": [
    "# Step 2:\n",
    "# Create the ViFi-CLIP models and load pretrained weights\n",
    "model = vificlip.returnCLIP(config,\n",
    "                            logger=logger,\n",
    "                            class_names=class_names,)\n",
    "model = model.float().cuda()  # changing to cuda here\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f5e321c8-8214-4010-8ee6-5d57def27c61",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-11-24 23:42:52 ViT-B/16](2665388570.py 1): INFO ==============> Resuming form /home/gamerx/Downloads/vifi_clip_10_epochs_k400_full_finetuned.pth....................\n"
     ]
    }
   ],
   "source": [
    "logger.info(f\"==============> Resuming form {config.MODEL.RESUME}....................\")\n",
    "checkpoint = torch.load(config.MODEL.RESUME, map_location='cpu')\n",
    "load_state_dict = checkpoint['model']\n",
    "# now remove the unwanted keys:\n",
    "if \"module.prompt_learner.token_prefix\" in load_state_dict:\n",
    "    del load_state_dict[\"module.prompt_learner.token_prefix\"]\n",
    "\n",
    "if \"module.prompt_learner.token_suffix\" in load_state_dict:\n",
    "    del load_state_dict[\"module.prompt_learner.token_suffix\"]\n",
    "\n",
    "if \"module.prompt_learner.complete_text_embeddings\" in load_state_dict:\n",
    "    del load_state_dict[\"module.prompt_learner.complete_text_embeddings\"]\n",
    "# create new OrderedDict that does not contain `module.`\n",
    "from collections import OrderedDict\n",
    "new_state_dict = OrderedDict()\n",
    "for k, v in load_state_dict.items():\n",
    "    name = k[7:] # remove `module.`\n",
    "    new_state_dict[name] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "13867cfa-876a-498e-b324-218e8a8bcdf4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-11-24 23:42:53 ViT-B/16](3852643250.py 3): INFO resume model: _IncompatibleKeys(missing_keys=['prompt_learner.complete_text_embeddings'], unexpected_keys=[])\n"
     ]
    }
   ],
   "source": [
    "# load params\n",
    "msg = model.load_state_dict(new_state_dict, strict=False)\n",
    "logger.info(f\"resume model: {msg}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e18b0f3f-541c-4009-945f-977cc4032f0d",
   "metadata": {},
   "source": [
    "### Preprocessing input video "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cfa85460-c71e-47be-8dbd-d3e181a633d8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Step 3: \n",
    "# Preprocessing for video\n",
    "img_norm_cfg = dict(\n",
    "    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_bgr=False)\n",
    "scale_resize = int(256 / 224 * config.DATA.INPUT_SIZE)\n",
    "val_pipeline = [\n",
    "    dict(type='DecordInit'),\n",
    "    dict(type='SampleFrames', clip_len=1, frame_interval=1, num_clips=config.DATA.NUM_FRAMES, test_mode=True),\n",
    "    dict(type='DecordDecode'),\n",
    "    dict(type='Resize', scale=(-1, scale_resize)),\n",
    "    dict(type='CenterCrop', crop_size=config.DATA.INPUT_SIZE),\n",
    "    dict(type='Normalize', **img_norm_cfg),\n",
    "    dict(type='FormatShape', input_format='NCHW'),\n",
    "    dict(type='Collect', keys=['imgs'], meta_keys=[]),\n",
    "    dict(type='ToTensor', keys=['imgs'])\n",
    "]\n",
    "if config.TEST.NUM_CROP == 3:\n",
    "    val_pipeline[3] = dict(type='Resize', scale=(-1, config.DATA.INPUT_SIZE))\n",
    "    val_pipeline[4] = dict(type='ThreeCrop', crop_size=config.DATA.INPUT_SIZE)\n",
    "if config.TEST.NUM_CLIP > 1:\n",
    "    val_pipeline[1] = dict(type='SampleFrames', clip_len=1, frame_interval=1, num_clips=config.DATA.NUM_FRAMES, multiview=config.TEST.NUM_CLIP)\n",
    "pipeline = Compose(val_pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4fb069f4-6adb-409e-936b-3b2992947d09",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_file = {'filename': video_path, 'tar': False, 'modality': 'RGB', 'start_index': 0}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed52bb28-c681-479a-89f3-fc86362b6d3f",
   "metadata": {},
   "source": [
    "### ViFi-CLIP inference with given video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9c7bd8f6-29f0-47bb-bc69-81fbc745a275",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "video = pipeline(dict_file)\n",
    "video_tensor = video['imgs'].unsqueeze(0).cuda().float()\n",
    "# Inference through ViFi-CLIP\n",
    "with torch.no_grad():\n",
    "    with torch.cuda.amp.autocast():\n",
    "        logits = model(video_tensor)\n",
    "pred_index = logits.argmax(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7737bbdf-ed86-400a-98f9-a2b36305eb6c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logits: tensor([[12.5938,  9.1016, 11.2266, 14.6953]], device='cuda:0',\n",
      "       dtype=torch.float16)\n",
      "predicted action category is : climbing stairs\n"
     ]
    }
   ],
   "source": [
    "print(f'logits: {logits}')\n",
    "print(f'predicted action category is : {class_names[pred_index]}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}