05_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To convert py from/to ipynb:\n",
    "\n",
    "```\n",
    "conda install conda install -c defaults -c conda-forge ipynb-py-convert\n",
    "```\n",
    "\n",
    "```\n",
    "ipynb-py-convert script.py script.ipynb\n",
    "ipynb-py-convert script.ipynb script.py\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.basics           import *\n",
    "from fastai.callback.all     import *\n",
    "from fastai.distributed      import *\n",
    "from fastai.tabular.all      import *\n",
    "\n",
    "import enum\n",
    "import gc\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import time\n",
    "\n",
    "from collections import defaultdict\n",
    "from pathlib import Path\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "Mode = enum.IntEnum('Mode', ['normal', 'hurry_up', 'blindfolded_gunslinger'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEVICE              = 'cuda'\n",
    "DO_NOT_UNTAR        = False\n",
    "MODE                = Mode.normal\n",
    "HURRY_UP_CUTOFF     = 0.25\n",
    "BLIND_CUTOFF        = 0.19\n",
    "PUB_PVT_CUTOFF      = 0.20\n",
    "TIME_BUDGET         = 8.75 * 60 * 60 # secs\n",
    "\n",
    "if MODE == Mode.normal or MODE == Mode.hurry_up:\n",
    "    ROWS_TO_INFER = 2.5e6\n",
    "elif MODE == Mode.blindfolded_gunslinger:\n",
    "    ROWS_TO_INFER = (1-BLIND_CUTOFF) * 2.5e6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class _H:\n",
    "    '''Hyperparams'''\n",
    "    def __init__(self, **kwargs):\n",
    "        self.__dict__ = kwargs\n",
    "\n",
    "    def __repr__(self):\n",
    "        return str(self.__dict__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "H = AttrDict(\n",
    "    {\n",
    "        'chunk_size': 500, \n",
    "        'bs': 64,\n",
    "        'valid_pct': 0.025,\n",
    "        'data': '210101b',\n",
    "    }\n",
    ")\n",
    "H1 = AttrDict(\n",
    "    {\n",
    "        'load': '210105_0.812534_relu_e3e3.pth',\n",
    "        'trf_dim': 512,\n",
    "        'trf_enc': 3,\n",
    "        'trf_dec': 3,\n",
    "        'trf_heads': 4, \n",
    "        'trf_do': 0.1, \n",
    "        'trf_act': 'relu', \n",
    "        'emb_do': 0.25, \n",
    "        'tfixup': True, \n",
    "    }\n",
    ")\n",
    "\n",
    "H2 = AttrDict(\n",
    "    {\n",
    "        'load': '210105_0.812154_gelu_e4d4_ep30.pth',\n",
    "        'data': '210101',\n",
    "        'trf_dim': 512,\n",
    "        'trf_enc': 4,\n",
    "        'trf_dec': 4,\n",
    "        'trf_heads': 4, \n",
    "        'trf_do': 0.1, \n",
    "        'trf_act': 'gelu', \n",
    "        'emb_do': 0.25, \n",
    "        'tfixup': True, \n",
    "    }\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KAGGLE = Path('/kaggle').exists()\n",
    "KAGGLE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Env dependent paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "if KAGGLE:\n",
    "    # Use kaggle test sets and force GPU + untar resources\n",
    "    ds_dir       = Path('/kaggle/input/riiid-acp')\n",
    "    DEVICE       = 'cuda'\n",
    "    DO_NOT_UNTAR = False\n",
    "else:\n",
    "    ds_dir       = Path('kaggle_dataset/to_upload')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_d = Path('resources')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unpack dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not DO_NOT_UNTAR:\n",
    "    if Path('resources').exists():\n",
    "        shutil.rmtree('resources')\n",
    "        \n",
    "    for tgz in ds_dir.glob('*.tgz'):\n",
    "        tgz = ds_dir / 'resources.tgz'\n",
    "        assert os.system(f'tar xvf {str(tgz)}') == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(in_d / f'data_{H.chunk_size}_last_interactions_v{H.data}.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fix typo\n",
    "data.last_q_container_id_d = data.last_q_contained_id_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "attempt_num = np.lib.format.open_memmap(in_d / f'data_attempt_num_v{H.data}.npy')\n",
    "attempts_correct = np.lib.format.open_memmap(in_d / f'data_attempts_correct_v{H.data}.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_list = sorted(data.cat_d.keys())\n",
    "users_d = defaultdict(lambda: len(users_d))\n",
    "for user_id in users_list:\n",
    "    users_d[user_id]\n",
    "assert len(users_d.keys()) == 393656\n",
    "assert users_d[2746] == 2\n",
    "assert users_d[2126571790] == 389719"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(in_d / f'meta_v{H.data}.pkl', 'rb') as f:\n",
    "    meta = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "Cats = enum.IntEnum('Cats', meta.cat_names, start=0)\n",
    "Conts = enum.IntEnum('Conts', meta.cont_names, start=0)\n",
    "QCols = enum.IntEnum('QCols', meta.qcols, start=0)\n",
    "LCols = enum.IntEnum('LCols', meta.lcols, start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TutorNet(nn.Module):\n",
    "    def __init__(self, emb_szs, tag_emb_szs, emb_do, n_cont, trf_dim, trf_enc, trf_dec, trf_heads, trf_do, trf_act):\n",
    "        super().__init__()\n",
    "        self.nhead,self.trf_dim = trf_heads, trf_dim\n",
    "        \n",
    "        tag_emb_szs =(tag_emb_szs[0]+1, trf_dim)\n",
    "\n",
    "        self.embeds    = nn.ModuleList([nn.Sequential(nn.Embedding(ni+1, nf, max_norm=1.),nn.Linear(nf,trf_dim)) \n",
    "                                        for ni,nf in emb_szs])\n",
    "        self.tagembeds = nn.EmbeddingBag(*tag_emb_szs, max_norm=1., mode='sum')\n",
    "            \n",
    "        self.conts     = nn.Linear(n_cont,trf_dim)\n",
    "            \n",
    "        self.trafo = nn.Transformer(\n",
    "            d_model = trf_dim,\n",
    "            nhead = trf_heads,\n",
    "            num_encoder_layers = trf_enc,\n",
    "            num_decoder_layers = trf_dec,\n",
    "            dim_feedforward = trf_dim*4,\n",
    "            dropout = trf_do,\n",
    "            activation = trf_act,\n",
    "        )\n",
    "\n",
    "        self.mlp = nn.Linear(trf_dim, 6)\n",
    "        \n",
    "    def forward(self, x_mask, x_cat, x_cont, x_tags, x_tagw, shuffle=None, lam=None):\n",
    "        b, sl, catf, contf, tagsf = (*x_cat.shape, x_cont.shape[2], x_tags.shape[2])\n",
    "        \n",
    "        x_cat  += 1\n",
    "        x_tags += 1\n",
    "    \n",
    "        # compute masks\n",
    "        causal_mask  = ~torch.tril(torch.ones(1,sl, sl,dtype=torch.bool,device=x_cat.device)).expand(b,-1,-1)\n",
    "        x_tci   = x_cat[...,Cats.task_container_id]\n",
    "        x_tci_s = torch.zeros_like(x_tci)\n",
    "        x_tci_s[...,1:] = x_tci[...,:-1]\n",
    "        enc_container_aware_mask =  (x_tci.unsqueeze(-1) == x_tci_s.unsqueeze(-1).permute(0,2,1)) | causal_mask\n",
    "        dec_container_aware_mask = ~(x_tci.unsqueeze(-1) == x_tci.unsqueeze(-1).permute(0,2,1))   & causal_mask\n",
    "\n",
    "        padding_mask = x_mask \n",
    "                \n",
    "        # encoder x (shifted q & a)\n",
    "        enc_cat  = torch.zeros_like(x_cat)\n",
    "        enc_cont = torch.zeros_like(x_cont)\n",
    "        enc_tags = torch.zeros_like(x_tags)\n",
    "        enc_tagw = torch.zeros_like(x_tagw)\n",
    "        \n",
    "        enc_cat[:,1:]  = x_cat[:,:-1]\n",
    "        enc_cont[:,1:] = x_cont[:,:-1]\n",
    "        enc_tags[:,1:] = x_tags[:,:-1]\n",
    "        enc_tagw[:,1:] = x_tagw[:,:-1]\n",
    "        \n",
    "        # decoder x (nonshifted q)\n",
    "        dec_cat  = x_cat\n",
    "        dec_cont = x_cont\n",
    "        dec_tags = x_tags\n",
    "        dec_tagw = x_tagw\n",
    "\n",
    "        # hide correct answer and user answered correctly from decoder\n",
    "        dec_cat[...,Cats.answered_correctly] = 0\n",
    "        dec_cat[...,Cats.user_answer] = 0\n",
    "        dec_cat[...,Cats.qhe] = 0\n",
    "        dec_cont[...,Conts.qet] = 0\n",
    "        dec_cont[...,Conts.qet_log] = 0\n",
    "        \n",
    "        # print(enc_cont.shape)\n",
    "        enc_cat  =  enc_cat.view(b * sl, catf)   # b*sl, catf\n",
    "        enc_tags = enc_tags.view(b * sl, tagsf) # b*sl, tagsf\n",
    "        enc_tagw = enc_tagw.view(b * sl, tagsf) # b*sl, tagsf\n",
    "\n",
    "        dec_cat  =  dec_cat.view(b * sl, catf)   # b*sl, catf\n",
    "        dec_tags = dec_tags.view(b * sl, tagsf) # b*sl, tagsf\n",
    "        dec_tagw = dec_tagw.view(b * sl, tagsf) # b*sl, tagsf\n",
    "        \n",
    "        # embed categorical vars\n",
    "        enc = torch.mean(torch.stack([\n",
    "            *[ e(enc_cat[:,i]) for i, e in enumerate(self.embeds) ],\n",
    "            self.tagembeds(enc_tags, per_sample_weights=enc_tagw),\n",
    "            self.conts(enc_cont).view(-1,self.trf_dim)\n",
    "        ]),dim=0)\n",
    "        \n",
    "        dec = torch.mean(torch.stack([\n",
    "            *[ e(dec_cat[:,i]) for i, e in enumerate(self.embeds) ],\n",
    "            self.tagembeds(dec_tags, per_sample_weights=dec_tagw),\n",
    "            self.conts(dec_cont).view(-1,self.trf_dim)\n",
    "        ]),dim=0)\n",
    "        \n",
    "        enc = enc.view(b, sl, self.trf_dim)           # b, sl, sum of cat, cont and tag ftrs\n",
    "        dec = dec.view(b, sl, self.trf_dim)           # b, sl, sum of cat, cont and tag ftrs\n",
    "\n",
    "        if shuffle is not None:\n",
    "            enc = torch.lerp(enc, enc[shuffle], lam.view(lam.shape[0], 1, 1))\n",
    "            dec = torch.lerp(dec, dec[shuffle], lam.view(lam.shape[0], 1, 1))\n",
    "            padding_mask = None\n",
    "            container_aware_mask |= container_aware_mask[shuffle]\n",
    "        \n",
    "        enc = enc.permute(1, 0, 2)          # sl, b, tf (torchformer input)\n",
    "        dec = dec.permute(1, 0, 2)          # sl, b, tf\n",
    "\n",
    "        expand_nheads = lambda t: t.unsqueeze(1).expand(t.shape[0],self.nhead,-1,-1).reshape(-1,*t.shape[-2:])\n",
    "        \n",
    "        o = self.trafo(\n",
    "            enc, \n",
    "            dec, \n",
    "            src_mask = expand_nheads(enc_container_aware_mask),\n",
    "            tgt_mask = expand_nheads(dec_container_aware_mask),\n",
    "            memory_mask = expand_nheads(enc_container_aware_mask),\n",
    "            src_key_padding_mask = padding_mask,\n",
    "            tgt_key_padding_mask = padding_mask,\n",
    "            memory_key_padding_mask = padding_mask,\n",
    "        )                                   # sl, b, tf\n",
    "        o = o.permute(1, 0, 2)              # b, sl, tf\n",
    "        o = self.mlp(o)                     # b, sl, of (of=2)\n",
    "        #print(o)\n",
    "        return o\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_szs = list(zip(meta.n_emb.values(), meta.emb_dim.values()))\n",
    "tag_emb_szs = meta.tags_n_emb, meta.tags_emb_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ModelPatcher:\n",
    "    def __init__(self):\n",
    "        self.patterns = []\n",
    "\n",
    "    def is_patchable(self, module_name, module, raiseError):\n",
    "        return True\n",
    "\n",
    "    def get_patchable_layers(self, model):\n",
    "        # Layer names (displayed as regexps)\")\n",
    "        ret = []\n",
    "        for k, v in model.named_modules():\n",
    "            if self.is_patchable(k, v, raiseError=False):\n",
    "                r = re.escape(k)\n",
    "                ret.append({\"regexp\": r, \"layer\": v})\n",
    "        return ret\n",
    "\n",
    "    def add_pattern(self, pattern, patch_info):\n",
    "        self.patterns.append(dict(pattern=pattern, patch_info=patch_info))\n",
    "\n",
    "    def pattern_match(self, module_name):\n",
    "        for pattern_def in self.patterns:\n",
    "            if re.match(pattern_def[\"pattern\"], module_name):\n",
    "                return True, pattern_def[\"patch_info\"]\n",
    "        return False, -1\n",
    "\n",
    "    def new_child_module(self, child_module_name, child_module, patch_info):\n",
    "        raise NotImplementedError(\"Implement this in subclasses\")\n",
    "\n",
    "    def replace_module(self, father, child_module_name, child_name, child_module, patch_info):\n",
    "        new_child_module = self.new_child_module(child_module_name, child_module, patch_info)\n",
    "        if new_child_module is not None:\n",
    "            setattr(father, child_name, new_child_module)\n",
    "\n",
    "    def patch_model(self, model):\n",
    "        modules = {}\n",
    "        modified = False\n",
    "        for k, v in model.named_modules():\n",
    "            modules[k] = v\n",
    "            match, patch_info = self.pattern_match(k)\n",
    "            if match and self.is_patchable(k, v, raiseError=True):\n",
    "                parts = k.split(\".\")\n",
    "                father_module_name = \".\".join(parts[:-1])\n",
    "                child_name = parts[-1]\n",
    "                father = modules[father_module_name]\n",
    "                self.replace_module(father, k, child_name, v, patch_info)\n",
    "                modified = True\n",
    "        if not modified:\n",
    "            print(\n",
    "                \"Warning: the patcher did not patch anything!\"\n",
    "                \" Check patchable layers with `mp.get_patchable_layers(model)`\"\n",
    "            )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = TutorNet(emb_szs, tag_emb_szs, H1.emb_do, len(meta.cont_names), H1.trf_dim, H1.trf_enc, H1.trf_dec, H1.trf_heads, H1.trf_do, H1.trf_act)\n",
    "model2 = TutorNet(emb_szs, tag_emb_szs, H2.emb_do, len(meta.cont_names), H2.trf_dim, H2.trf_enc, H2.trf_dec, H2.trf_heads, H2.trf_do, H2.trf_act)\n",
    "\n",
    "def trunc_normal_(x, mean=0., std=1.):\n",
    "    \"Truncated normal initialization (approximation)\"\n",
    "    # From https://discuss.pytorch.org/t/implementing-truncated-normal-initializer/4778/12\n",
    "    return x.normal_().fmod_(2).mul_(std).add_(mean)\n",
    "\n",
    "class MyModelPatcher(ModelPatcher):\n",
    "    def new_child_module(self, child_module_name, child_module, patch_info): return nn.Identity()\n",
    "mp = MyModelPatcher()\n",
    "mp.add_pattern(r\".*norm\\d?.*\",{})\n",
    "    \n",
    "if H1.tfixup: mp.patch_model(model1)\n",
    "if H2.tfixup: mp.patch_model(model2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict1 = torch.load(in_d / f'{H1.load}', map_location=DEVICE)\n",
    "state_dict2 = torch.load(in_d / f'{H2.load}', map_location=DEVICE)\n",
    "# %%\n",
    "if 'model' in state_dict1:\n",
    "    state_dict1 = state_dict1['model']\n",
    "if 'model' in state_dict2:\n",
    "    state_dict2 = state_dict2['model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = model1.to(DEVICE)\n",
    "model2 = model2.to(DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.load_state_dict(state_dict1)\n",
    "model2.load_state_dict(state_dict2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Infer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyRiiidEnv:\n",
    "    def __init__(self, p):\n",
    "        test_dtypes = {\n",
    "            'group_num': 'int64',\n",
    "            'row_id': 'int64',\n",
    "            'timestamp': 'int64',\n",
    "            'user_id': 'int32',\n",
    "            'content_id': 'int16',\n",
    "            'content_type_id': 'int8',\n",
    "            'task_container_id': 'int16',\n",
    "            'prior_question_elapsed_time': 'float32',\n",
    "            'prior_question_had_explanation': 'boolean',\n",
    "            'prior_group_answers_correct': 'object',\n",
    "            'prior_group_responses': 'object',\n",
    "        }\n",
    "    \n",
    "        pred_dtypes = {\n",
    "            'group_num': 'int64',\n",
    "            'row_id': 'int64',\n",
    "            'answered_correctly': 'float64',\n",
    "        }\n",
    "\n",
    "        self.test_df = pd.read_csv(\n",
    "            p / f'validation_x_{H.valid_pct}.csv',\n",
    "            usecols=test_dtypes.keys(),\n",
    "            dtype=test_dtypes,\n",
    "        ).set_index('group_num')\n",
    "        \n",
    "        self.pred_df = pd.read_csv(\n",
    "            p / f'validation_submission_{H.valid_pct}.csv',\n",
    "            usecols=pred_dtypes.keys(),\n",
    "            dtype=pred_dtypes,\n",
    "        ).set_index('group_num')\n",
    "        \n",
    "        self.first = True\n",
    "\n",
    "    def iter_test(self):\n",
    "        for (_, t), (_, p) in zip(self.test_df.groupby('group_num'), self.pred_df.groupby('group_num')):\n",
    "            yield t, p\n",
    "\n",
    "    def predict(self, p):\n",
    "        if self.first:\n",
    "            p.to_csv('submission.csv', index=False)\n",
    "            self.first = False\n",
    "        else:\n",
    "            p.to_csv('submission.csv', index=False, mode='a', header=False)\n",
    "\n",
    "def make_env():\n",
    "    return MyRiiidEnv(Path('/space/riiid-acp'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "if KAGGLE:\n",
    "    import riiideducation\n",
    "    env = riiideducation.make_env()\n",
    "else:\n",
    "    env = make_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_questions(df, Col, cat_names, cont_names, qc_d, lc_d, codes_d, QCols, LCols, Cats, Conts, \n",
    "        hist_cat_d, hist_cont_d, hist_tags_d, hist_tagw_d, last_q_container_d, last_ts, attempt_num, \n",
    "        attempts_correct, qp_d, users_d):\n",
    "    \n",
    "    df_a = df.values\n",
    "    \n",
    "    n_rows = len(df)\n",
    "    \n",
    "    # Prefetch tslis per (user_id, tcid) for better_tsli calculation\n",
    "    # NOTE the keys (user_id, tcid) are NOT encoded\n",
    "    tsli_d = defaultdict(list)\n",
    "    #for i, (_, row) in enumerate(df_d.items()): # SLOW\n",
    "    #for i, (_, row) in enumerate(df.iterrows()): # SUPER SLOW\n",
    "    for i, row in enumerate(df_a):\n",
    "        user_id, tcid, ts = row[Col.user_id], row[Col.task_container_id], row[Col.timestamp]\n",
    "        encoded_user_id = users_d[user_id]\n",
    "        tsli_d[user_id, tcid].append(ts - last_ts[encoded_user_id,0])\n",
    "        last_ts[encoded_user_id,0] = np.int64(ts)\n",
    "        \n",
    "    # average all tslis in the same task container\n",
    "    tsli_d = { k: sum(v)/len(v) for k, v in tsli_d.items() }\n",
    "    \n",
    "    # append df data to history\n",
    "    for i, row in enumerate(df_a):\n",
    "        user_id = row[Col.user_id]\n",
    "        encoded_user_id = users_d[user_id]\n",
    "        user_has_hist = user_id in hist_cat_d\n",
    "        if user_has_hist:\n",
    "            h_cat  = hist_cat_d [user_id] # just shortcuts\n",
    "            h_cont = hist_cont_d[user_id]\n",
    "            h_tags = hist_tags_d[user_id]\n",
    "            h_tagw = hist_tagw_d[user_id]\n",
    "        \n",
    "        cat  = np.zeros(len(cat_names),  dtype=np.int16)\n",
    "        cont = np.full (len(cont_names), np.nan, dtype=np.float32)\n",
    "\n",
    "        # Categorical test data\n",
    "        content_id = row[Col.content_id]\n",
    "        is_question = row[Col.content_type_id] == 0\n",
    "\n",
    "        if is_question:\n",
    "            encoded_question_id = codes_d['question_id'][content_id]\n",
    "            qc_row = qc_d[encoded_question_id]\n",
    "            cat[Cats.bundle_id]        = qc_row[QCols.bundle_id]\n",
    "            cat[Cats.correct_answer]   = qc_row[QCols.correct_answer]\n",
    "            cat[Cats.part]             = qc_row[QCols.part]\n",
    "            cat[Cats.question_id]      = encoded_question_id\n",
    "            cat[Cats.already_answered] = (int)(attempt_num[encoded_user_id, encoded_question_id-1] > 0)\n",
    "            cat[Cats.qhe]              = 0  # question has explanation?, not known yet    \n",
    "        else:\n",
    "            encoded_lecture_id = codes_d['lecture_id'][content_id]\n",
    "            lc_row = lc_d[encoded_lecture_id]\n",
    "            cat[Cats.lecture_id] = encoded_lecture_id\n",
    "            cat[Cats.part]       = lc_row[LCols.part]\n",
    "            cat[Cats.type_of]    = lc_row[LCols.type_of]\n",
    "\n",
    "        tcid = row[Col.task_container_id]\n",
    "        encoded_pqhe = codes_d['prior_question_had_explanation'][row[Col.prior_question_had_explanation]]\n",
    "        encoded_tcid = codes_d['task_container_id'][tcid]\n",
    "        cat[Cats.task_container_id] = encoded_tcid\n",
    "        \n",
    "        # Continuous test data\n",
    "        ts = row[Col.timestamp]\n",
    "        ts_mod_1day = ts % (1000 * 60 * 60 * 24)\n",
    "        ts_mod_1week = ts % (1000 * 60 * 60 * 24 * 7)\n",
    "        pqet = row[Col.prior_question_elapsed_time]\n",
    "        tsli = tsli_d[(user_id, tcid)] if user_has_hist else np.nan\n",
    "        clipped_tsli = min(tsli, 1000 * 60 * 20) # 20 minutes\n",
    "        \n",
    "        cont[Conts.qet]              = np.nan\n",
    "        cont[Conts.timestamp]        = ts\n",
    "        cont[Conts.tsli]             = tsli\n",
    "        cont[Conts.clipped_tsli]     = clipped_tsli\n",
    "        cont[Conts.qet_log]          = np.nan\n",
    "        cont[Conts.timestamp_log]    = np.log1p(ts)\n",
    "        cont[Conts.tsli_log]         = np.log1p(tsli)\n",
    "        cont[Conts.clipped_tsli_log] = np.log1p(clipped_tsli)\n",
    "        cont[Conts.ts_mod_1day]      = ts_mod_1day\n",
    "        cont[Conts.ts_mod_1day_sin]  = np.sin(ts_mod_1day * 2 * np.pi / (1000 * 60 * 60 * 24))\n",
    "        cont[Conts.ts_mod_1day_cos]  = np.cos(ts_mod_1day * 2 * np.pi / (1000 * 60 * 60 * 24))\n",
    "        cont[Conts.ts_mod_1week]     = ts_mod_1week\n",
    "        cont[Conts.ts_mod_1week_sin] = np.sin(ts_mod_1week * 2 * np.pi / (1000 * 60 * 60 * 24 * 7))\n",
    "        cont[Conts.ts_mod_1week_cos] = np.cos(ts_mod_1week * 2 * np.pi / (1000 * 60 * 60 * 24 * 7))\n",
    "        \n",
    "        # container ordinal\n",
    "        if user_has_hist and h_cat[-1,Cats.task_container_id] == encoded_tcid:\n",
    "            cont[Conts.container_ord] = h_cont[-1,Conts.container_ord] + 1\n",
    "        else:\n",
    "            cont[Conts.container_ord] = 0\n",
    "        \n",
    "        if is_question:\n",
    "            # Update qet and qet_log in history (make qet in last bundle skipping lectures = pqet)\n",
    "            if user_id in last_q_container_d and encoded_tcid != last_q_container_d[user_id]:\n",
    "                idx = h_cat[:,Cats.task_container_id] == last_q_container_d[user_id]\n",
    "                h_cat [idx,Cats.qhe]      = encoded_pqhe\n",
    "                h_cont[idx,Conts.qet]     = pqet\n",
    "                h_cont[idx,Conts.qet_log] = np.log1p(pqet)\n",
    "                        \n",
    "            last_q_container_d[user_id] = encoded_tcid\n",
    "            \n",
    "            # Update attempt_num\n",
    "            an = attempt_num     [encoded_user_id, encoded_question_id-1] # np.uint8\n",
    "            ac = attempts_correct[encoded_user_id, encoded_question_id-1] # np.uint8\n",
    "            cont[Conts.attempt_num]              = an\n",
    "            cont[Conts.attempt_num_log]          = np.log1p(an)\n",
    "            \n",
    "            # Update attempts_correct with what we know so far (will be re-updated after we've got the answers)\n",
    "            cont[Conts.attempts_correct]         = ac\n",
    "            cont[Conts.attempts_correct_log]     = np.log1p(ac)\n",
    "            if an != 0:\n",
    "                cont[Conts.attempts_correct_avg]     = ac / an\n",
    "                cont[Conts.attempts_correct_avg_log] = np.log1p(ac / an)\n",
    "\n",
    "            attempt_num[encoded_user_id, encoded_question_id-1] += np.uint8(1)\n",
    "\n",
    "            # question occurrence prob\n",
    "            cont[Conts.qp]              = qp_d[content_id] # qp_d indexes are non-encoded qids\n",
    "            cont[Conts.qp_log]          = np.log1p(cont[Conts.qp])\n",
    "\n",
    "        # Tags and weights\n",
    "        if is_question:\n",
    "            tags = qc_row[[ QCols.tag_0, QCols.tag_1, QCols.tag_2, QCols.tag_3, QCols.tag_4, QCols.tag_5 ]]\n",
    "        else:\n",
    "            tags = lc_row[[ LCols.tag_0, LCols.tag_1, LCols.tag_2, LCols.tag_3, LCols.tag_4, LCols.tag_5 ]]\n",
    "        tags = tags.astype(np.uint8)\n",
    "        tagw = (tags != 0).astype(np.float16)\n",
    "        sums = tagw.sum()\n",
    "        if sums > 0:\n",
    "            tagw /= sums\n",
    "       \n",
    "        # Concat history and new test data\n",
    "        if user_has_hist:\n",
    "            hist_cat_d [user_id] = np.concatenate((h_cat,  np.expand_dims(cat,  0)))\n",
    "            hist_cont_d[user_id] = np.concatenate((h_cont, np.expand_dims(cont, 0)))\n",
    "            hist_tags_d[user_id] = np.concatenate((h_tags, np.expand_dims(tags, 0)))\n",
    "            hist_tagw_d[user_id] = np.concatenate((h_tagw, np.expand_dims(tagw, 0)))\n",
    "        else:\n",
    "            hist_cat_d [user_id] = np.expand_dims(cat,  0)\n",
    "            hist_cont_d[user_id] = np.expand_dims(cont, 0)\n",
    "            hist_tags_d[user_id] = np.expand_dims(tags, 0)\n",
    "            hist_tagw_d[user_id] = np.expand_dims(tagw, 0)\n",
    "\n",
    "    return df.user_id.values\n",
    "\n",
    "\n",
    "def update_answers(prior_user_ids, prior_group_answers_correct, prior_group_responses, \n",
    "        cat_names, cont_names, codes_d, hist_cat_d, hist_cont_d, users_d, attempt_num, attempts_correct):\n",
    "\n",
    "    idx_per_uid_d = defaultdict(int)\n",
    "    for uid in prior_user_ids:\n",
    "        idx_per_uid_d[uid] -= 1\n",
    "\n",
    "    for i, uid in enumerate(prior_user_ids):\n",
    "        h_cat  = hist_cat_d [uid] # just shortcuts\n",
    "        h_cont = hist_cont_d[uid]\n",
    "        \n",
    "        idx = idx_per_uid_d[uid]\n",
    "        idx_per_uid_d[uid] += 1\n",
    "        \n",
    "        # Update categorical vars\n",
    "        h_cat [idx,Cats.answered_correctly] = codes_d['answered_correctly'][prior_group_answers_correct[i]]\n",
    "        h_cat [idx,Cats.user_answer]        = codes_d['user_answer'][prior_group_responses[i]]\n",
    "\n",
    "        # Update continuous vars\n",
    "        eqid = h_cat[idx,Cats.question_id]\n",
    "        if eqid > 0: # it's a question\n",
    "            assert prior_group_answers_correct[i] >= 0\n",
    "            euid = users_d[uid]\n",
    "            ac = attempts_correct[euid,eqid-1] # np.int8\n",
    "            an = h_cont[idx,Conts.attempt_num] # np.float32\n",
    "            h_cont[idx,Conts.attempts_correct]         = ac\n",
    "            h_cont[idx,Conts.attempts_correct_log]     = np.log1p(ac)\n",
    "            if an != 0:\n",
    "                h_cont[idx,Conts.attempts_correct_avg]     = ac / an\n",
    "                h_cont[idx,Conts.attempts_correct_avg_log] = np.log1p(ac / an)\n",
    "\n",
    "            attempts_correct[euid,eqid-1] = ac + np.uint8(prior_group_answers_correct[i])\n",
    "        else:\n",
    "            assert prior_group_answers_correct[i] == -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_x(user_ids, cat_names, cont_names, hist_cat_d, hist_cont_d, hist_tags_d, hist_tagw_d, chunk_size):\n",
    "    num_rows_per_uid_d = defaultdict(int)\n",
    "    for uid in user_ids:\n",
    "        num_rows_per_uid_d[uid] += 1\n",
    "\n",
    "    n_users = len(num_rows_per_uid_d)\n",
    "\n",
    "    # prepare the batch\n",
    "    x_mask = np.ones ((n_users, chunk_size), dtype=np.bool)\n",
    "    x_cat  = np.zeros((n_users, chunk_size, len(cat_names)),  dtype=np.long)\n",
    "    x_cont = np.full ((n_users, chunk_size, len(cont_names)), np.nan, dtype=np.float32)\n",
    "    x_tags = np.zeros((n_users, chunk_size, 6), dtype=np.long)\n",
    "    x_tagw = np.zeros((n_users, chunk_size, 6), dtype=np.float32)\n",
    "    \n",
    "    for i, uid in enumerate(num_rows_per_uid_d.keys()):\n",
    "        # trim history\n",
    "        hist_cat_d [uid] = hist_cat_d [uid][-chunk_size:]\n",
    "        hist_cont_d[uid] = hist_cont_d[uid][-chunk_size:]\n",
    "        hist_tags_d[uid] = hist_tags_d[uid][-chunk_size:]\n",
    "        hist_tagw_d[uid] = hist_tagw_d[uid][-chunk_size:]\n",
    "\n",
    "        sl = hist_cat_d[uid].shape[0]\n",
    "        \n",
    "        x_mask[i,:sl] = False\n",
    "        x_cat [i,:sl] = hist_cat_d[uid]\n",
    "        x_cont[i,:sl] = hist_cont_d[uid]\n",
    "        x_tags[i,:sl] = hist_tags_d[uid]\n",
    "        x_tagw[i,:sl] = hist_tagw_d[uid]\n",
    "    \n",
    "    return x_mask, x_cat, x_cont, x_tags, x_tagw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_preds(user_ids, preds, x_mask):\n",
    "    poi = torch.zeros(len(user_ids), 2, device=preds.device) # predictions of interest\n",
    "\n",
    "    user_row = defaultdict(lambda: len(user_row))\n",
    "    for uid in user_ids:\n",
    "        user_row[uid]\n",
    "\n",
    "    poi_idxs = { uid: torch.from_numpy(user_ids == uid)  for uid in user_row.keys() }\n",
    "    \n",
    "    for uid in user_ids:\n",
    "        ur = user_row[uid]           # user row (1st dim) of the preds tensor\n",
    "        pi = poi_idxs[uid]           # indexes to the original locations of the interactions\n",
    "        x = preds[ur,~x_mask[ur],:2] # get all predictions (both history and new)\n",
    "        x = x[-pi.sum():]            # the last pi.sum() preds are the new ones\n",
    "        poi[pi] = x\n",
    "        \n",
    "    return torch.softmax(poi, dim=-1)[:,1].detach().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numba import njit\n",
    "from scipy.stats import rankdata\n",
    "\n",
    "@njit\n",
    "def _auc(actual, pred_ranks):\n",
    "    actual = np.asarray(actual)\n",
    "    pred_ranks = np.asarray(pred_ranks)\n",
    "    n_pos = np.sum(actual)\n",
    "    n_neg = len(actual) - n_pos\n",
    "    return (np.sum(pred_ranks[actual==1]) - n_pos*(n_pos+1)/2) / (n_pos*n_neg)\n",
    "\n",
    "def auc(actual, predicted):\n",
    "    pred_ranks = rankdata(predicted)\n",
    "    return _auc(actual, pred_ranks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_roc_auc(pred, targ):\n",
    "    idx = targ != -1\n",
    "    pred = pred[idx]\n",
    "    targ = targ[idx]\n",
    "    return auc(targ.cpu().numpy(), pred.cpu().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "previous_users = None\n",
    "means = torch.from_numpy(meta.means).to(DEVICE)\n",
    "stds  = torch.from_numpy(meta.stds).to(DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not KAGGLE:\n",
    "    # delete history of validation users\n",
    "    val_users = users_list[int((1 - H.valid_pct) * len(users_list)):]\n",
    "    for val_user in val_users:\n",
    "        encoded_val_user = users_d[val_user]\n",
    "        if val_user in data.cat_d:\n",
    "            del data.cat_d[val_user]\n",
    "            del data.cont_d[val_user]\n",
    "            del data.tags_d[val_user]\n",
    "            del data.tagw_d[val_user]\n",
    "            del data.last_q_container_id_d[val_user]\n",
    "            data.last_ts[encoded_val_user,0] = 0\n",
    "            attempt_num[encoded_val_user] = 0\n",
    "            attempts_correct[encoded_val_user] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = model1.eval()\n",
    "model2 = model2.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6425it [4:14:16,  2.56s/it, model 1=1636752, model 1+2=1636691, eta=6.473/8.726, auroc (pub)=0.816131, auroc (pvt)=0.815113]IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "10000it [6:32:02,  2.35s/it, model 1=2482136, model 1+2=2482075, eta=6.581/8.726, auroc (pub)=0.816131, auroc (pvt)=0.816148]\n"
     ]
    }
   ],
   "source": [
    "n_read_rows = 0\n",
    "n_predicted_rows = 0\n",
    "n_predicted_rows_by_model_2 = 0\n",
    "inference_start_time = time.time()\n",
    "flag_ensemble = True\n",
    "\n",
    "Col = None\n",
    "prior_user_ids = None # linter go away\n",
    "all_preds = torch.FloatTensor()\n",
    "all_targs = torch.LongTensor()\n",
    "\n",
    "pbar = tqdm(env.iter_test())\n",
    "\n",
    "for test_df, pred_df in pbar:\n",
    "    if Col is None:\n",
    "        Col = enum.IntEnum('Col', test_df.columns.tolist(), start=0)\n",
    "\n",
    "    prior_group_answers_correct = np.fromstring(test_df.iloc[0].prior_group_answers_correct[1:-1], dtype=np.int16, sep=',')\n",
    "    prior_group_responses       = np.fromstring(test_df.iloc[0].prior_group_responses      [1:-1], dtype=np.int16, sep=',')\n",
    "\n",
    "    if MODE == Mode.hurry_up and n_read_rows > HURRY_UP_CUTOFF * 2.5e6:\n",
    "        preds = torch.full((len(test_df),), 0.5)\n",
    "    else:\n",
    "        if prior_group_responses.size > 0: update_answers(\n",
    "            prior_user_ids,\n",
    "            prior_group_answers_correct,\n",
    "            prior_group_responses,\n",
    "            meta.cat_names,\n",
    "            meta.cont_names,\n",
    "            meta.codes_d,\n",
    "            data.cat_d,\n",
    "            data.cont_d,\n",
    "            users_d,\n",
    "            attempt_num,\n",
    "            attempts_correct\n",
    "        )\n",
    "\n",
    "        prior_user_ids = update_questions(\n",
    "            test_df, \n",
    "            Col,\n",
    "            meta.cat_names, \n",
    "            meta.cont_names, \n",
    "            meta.qc_d, \n",
    "            meta.lc_d,\n",
    "            meta.codes_d, \n",
    "            QCols, \n",
    "            LCols, \n",
    "            Cats,\n",
    "            Conts, \n",
    "            data.cat_d, \n",
    "            data.cont_d, \n",
    "            data.tags_d, \n",
    "            data.tagw_d, \n",
    "            data.last_q_container_id_d,\n",
    "            data.last_ts, \n",
    "            attempt_num,\n",
    "            attempts_correct,\n",
    "            data.qp_d,\n",
    "            users_d\n",
    "        )\n",
    "            \n",
    "        # get x\n",
    "        a_mask, a_cat, a_cont, a_tags, a_tagw = get_x(\n",
    "            prior_user_ids,\n",
    "            meta.cat_names,\n",
    "            meta.cont_names,\n",
    "            data.cat_d,\n",
    "            data.cont_d,\n",
    "            data.tags_d,\n",
    "            data.tagw_d,\n",
    "            H.chunk_size\n",
    "        )\n",
    "\n",
    "        # Predict\n",
    "        if MODE == Mode.blindfolded_gunslinger and n_read_rows < BLIND_CUTOFF * 2.5e6:\n",
    "            preds = torch.full((len(test_df),), 0.5)\n",
    "            inference_start_time = time.time()\n",
    "        else:\n",
    "            with torch.no_grad():\n",
    "                batch_preds1 = torch.FloatTensor().to(DEVICE)\n",
    "                if flag_ensemble:\n",
    "                    batch_preds2 = torch.FloatTensor().to(DEVICE)\n",
    "                for b in range((a_cat.shape[0] + H.bs - 1) // H.bs):\n",
    "                    x_mask = torch.from_numpy(a_mask[b*H.bs:(b+1)*H.bs]).to(DEVICE)\n",
    "                    x_cat  = torch.from_numpy(a_cat [b*H.bs:(b+1)*H.bs]).to(DEVICE)\n",
    "                    x_cont = torch.from_numpy(a_cont[b*H.bs:(b+1)*H.bs]).to(DEVICE)\n",
    "                    x_tags = torch.from_numpy(a_tags[b*H.bs:(b+1)*H.bs]).to(DEVICE)\n",
    "                    x_tagw = torch.from_numpy(a_tagw[b*H.bs:(b+1)*H.bs]).to(DEVICE)\n",
    "\n",
    "                    # Normalize x_cont on GPU and take care of nans\n",
    "                    x_cont = (x_cont - means) / stds\n",
    "                    x_cont[torch.isnan(x_cont)] = 0.\n",
    "                    x_cont = x_cont.to(torch.float32)\n",
    "\n",
    "                    if flag_ensemble:\n",
    "                        preds1 = model1(x_mask.clone(), x_cat.clone(), x_cont.clone(), x_tags.clone(), x_tagw.clone())\n",
    "                    else:\n",
    "                        preds1 = model1(x_mask, x_cat, x_cont, x_tags, x_tagw)\n",
    "                    batch_preds1 = torch.cat([batch_preds1, preds1])\n",
    "\n",
    "                    if flag_ensemble:\n",
    "                        preds2 = model2(x_mask, x_cat, x_cont, x_tags, x_tagw)\n",
    "                        batch_preds2 = torch.cat([batch_preds2, preds2])\n",
    "\n",
    "                preds = get_preds(prior_user_ids, batch_preds1, torch.from_numpy(a_mask).to(DEVICE))\n",
    "\n",
    "                n_predicted_rows += len(test_df)\n",
    "                if flag_ensemble:\n",
    "                    preds2 = get_preds(prior_user_ids, batch_preds2, torch.from_numpy(a_mask).to(DEVICE))\n",
    "                    preds = (preds + preds2) / 2\n",
    "                    n_predicted_rows_by_model_2 += len(test_df)\n",
    "\n",
    "    test_df['answered_correctly'] = preds\n",
    "    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n",
    "\n",
    "    # adaptive ensembling\n",
    "    n_read_rows += len(test_df)\n",
    "    if n_predicted_rows < 1000:\n",
    "        flag_ensemble = True\n",
    "    else:\n",
    "        elapsed_inference_time = time.time() - inference_start_time\n",
    "        estimated_total_inference_time = elapsed_inference_time * ROWS_TO_INFER / n_predicted_rows\n",
    "        startup_time = inference_start_time - start_time\n",
    "        flag_ensemble = estimated_total_inference_time < (TIME_BUDGET - startup_time)\n",
    "\n",
    "    if not KAGGLE:\n",
    "        all_preds = torch.cat([all_preds, preds])\n",
    "        all_targs = torch.cat([all_targs, torch.LongTensor(prior_group_answers_correct)])\n",
    "        pub_preds = all_preds[:int(PUB_PVT_CUTOFF * 2.5e6)]\n",
    "        pub_targs = all_targs[:int(PUB_PVT_CUTOFF * 2.5e6)]\n",
    "        pvt_preds = all_preds[int(PUB_PVT_CUTOFF * 2.5e6):]\n",
    "        pvt_targs = all_targs[int(PUB_PVT_CUTOFF * 2.5e6):]\n",
    "        postfix = {\n",
    "            'model 1': n_predicted_rows,\n",
    "            'model 1+2': n_predicted_rows_by_model_2,\n",
    "        }\n",
    "        if n_predicted_rows >= 1000:\n",
    "            postfix['eta'] = f'{estimated_total_inference_time / 60 / 60:.3f}/{(TIME_BUDGET - startup_time) / 60 / 60:.3f}'\n",
    "\n",
    "        if len(pub_targs) > 0:\n",
    "            pub_auroc = my_roc_auc(pub_preds[:len(pub_targs)], pub_targs)\n",
    "            postfix['auroc (pub)'] = f'{pub_auroc:.6f}'\n",
    "        if len(pvt_targs) > 0:\n",
    "            pvt_auroc = my_roc_auc(pvt_preds[:len(pvt_targs)], pvt_targs)\n",
    "            postfix['auroc (pvt)'] = f'{pvt_auroc:.6f}'\n",
    "        pbar.set_postfix(postfix)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "if KAGGLE:\n",
    "    shutil.rmtree('resources')\n"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}