T5chem_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports and Const"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "where is the data located"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "based_data_path = '/home/eyal/Desktop/University/Research/DeepWithdrawn/split/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prep Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This step converts our data to the format t5chem expects to receive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(scheme, modal):\n",
    "    train_data_path = os.path.join(based_data_path, scheme, modal, 'train2.csv')\n",
    "    val_data_path = os.path.join(based_data_path, scheme, modal, 'val.csv')\n",
    "    test_data_path = os.path.join(based_data_path, scheme, modal, 'test.csv')\n",
    "    \n",
    "    train_df = pd.read_csv(train_data_path, usecols=['smiles', 'withdrawn_class'])\n",
    "    train_df['type'] = 'train'\n",
    "\n",
    "    val_df = pd.read_csv(val_data_path, usecols=['smiles', 'withdrawn_class'])\n",
    "    val_df['type'] = 'val'\n",
    "\n",
    "    test_df = pd.read_csv(test_data_path, usecols=['smiles', 'withdrawn_class'])\n",
    "    test_df['type'] = 'test'\n",
    "\n",
    "    raw_df = pd.concat([train_df, val_df, test_df])\n",
    "    \n",
    "    return raw_df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_input(row, source_file, target_file):\n",
    "    print(row['smiles'], file=source_file)\n",
    "    print(row['withdrawn_class'], file=target_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "db_agree_dups\n",
      "db_agree_no_dups\n",
      "db_no_agree_dups\n",
      "db_no_agree_no_dups\n"
     ]
    }
   ],
   "source": [
    "for folder in ['db_agree_dups', 'db_agree_no_dups', 'db_no_agree_dups', 'db_no_agree_no_dups']:\n",
    "    print(folder)\n",
    "    for modal in ['DrugBank', 'ChEMBL', 'NCATS']:\n",
    "        raw_df = load_data(folder, modal)\n",
    "        input_source_dict = {}\n",
    "        input_target_dict = {}\n",
    "        input_folder_path = os.path.join(\"./data/drug_withdrawn\", folder, modal)\n",
    "        os.makedirs(input_folder_path)\n",
    "        for data_type in ['train', 'val', 'test']:\n",
    "            input_source_dict[data_type] = open(os.path.join(input_folder_path, data_type + '.source'), \"w\")\n",
    "            input_target_dict[data_type] = open(os.path.join(input_folder_path, data_type + '.target'), \"w\")\n",
    "        for i,item in raw_df.iterrows():\n",
    "            data_type = item['type']\n",
    "            prepare_input(item, input_source_dict[data_type], input_target_dict[data_type])\n",
    "\n",
    "        for key, val in input_source_dict.items():\n",
    "            val.close()\n",
    "        for key,val in input_target_dict.items():\n",
    "            val.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "here we point to the path of the pre-trained model\n",
    "and then train t5chem for each evaluation and each test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pre_train_model = './models/pretrain/simple/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of T5ForProperty were not initialized from the model checkpoint at ./models/pretrain/simple/ and are newly initialized: ['lm_head.0.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/home/eyal/.local/lib/python3.10/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 4341\n",
      "  Num Epochs = 100\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 13600\n",
      "  Number of trainable parameters = 14840308\n",
      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
      "wandb: Currently logged in as: eyalmazuz. Use `wandb login --relogin` to force relogin\n",
      "wandb: ERROR Failed to sample metric: Not Supported\n",
      "wandb: Tracking run with wandb version 0.13.5\n",
      "wandb: Run data is saved locally in /home/eyal/Desktop/University/Research/DeepWithdrawn/wandb/run-20221204_171307-2bz3owic\n",
      "wandb: Run `wandb offline` to turn off syncing.\n",
      "wandb: Syncing run ./model/db_agree_dups/DrugBank\n",
      "wandb: ⭐️ View project at https://wandb.ai/eyalmazuz/huggingface\n",
      "wandb: 🚀 View run at https://wandb.ai/eyalmazuz/huggingface/runs/2bz3owic\n",
      "  0%|          | 0/13600 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(data_dir='./data/drug_withdrawn/db_agree_dups/DrugBank', output_dir='./model/db_agree_dups/DrugBank', task_type='classification', pretrain='./models/pretrain/simple/', vocab='', tokenizer='', random_seed=8570, num_epoch=100, log_step=5000, batch_size=32, init_lr=0.0005, num_classes=2, command=<function train at 0x7f2406e2d510>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 42/13600 [00:10<58:53,  3.84it/s]  \n",
      "wandb: Waiting for W&B process to finish... (success).\n",
      "wandb: Synced ./model/db_agree_dups/DrugBank: https://wandb.ai/eyalmazuz/huggingface/runs/2bz3owic\n",
      "wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
      "wandb: Find logs at: ./wandb/run-20221204_171307-2bz3owic/logs\n",
      "Some weights of T5ForProperty were not initialized from the model checkpoint at ./models/pretrain/simple/ and are newly initialized: ['lm_head.0.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/home/eyal/.local/lib/python3.10/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 4169\n",
      "  Num Epochs = 100\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 13100\n",
      "  Number of trainable parameters = 14840308\n",
      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
      "wandb: Currently logged in as: eyalmazuz. Use `wandb login --relogin` to force relogin\n",
      "wandb: ERROR Failed to sample metric: Not Supported\n",
      "wandb: Tracking run with wandb version 0.13.5\n",
      "wandb: Run data is saved locally in /home/eyal/Desktop/University/Research/DeepWithdrawn/wandb/run-20221204_171335-gwq8x215\n",
      "wandb: Run `wandb offline` to turn off syncing.\n",
      "wandb: Syncing run ./model/db_agree_dups/ChEMBL\n",
      "wandb: ⭐️ View project at https://wandb.ai/eyalmazuz/huggingface\n",
      "wandb: 🚀 View run at https://wandb.ai/eyalmazuz/huggingface/runs/gwq8x215\n",
      "  0%|          | 0/13100 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(data_dir='./data/drug_withdrawn/db_agree_dups/ChEMBL', output_dir='./model/db_agree_dups/ChEMBL', task_type='classification', pretrain='./models/pretrain/simple/', vocab='', tokenizer='', random_seed=8570, num_epoch=100, log_step=5000, batch_size=32, init_lr=0.0005, num_classes=2, command=<function train at 0x7f6751321510>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 22/13100 [00:06<1:01:44,  3.53it/s]\n",
      "wandb: Waiting for W&B process to finish... (success).\n",
      "wandb: Synced ./model/db_agree_dups/ChEMBL: https://wandb.ai/eyalmazuz/huggingface/runs/gwq8x215\n",
      "wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
      "wandb: Find logs at: ./wandb/run-20221204_171335-gwq8x215/logs\n"
     ]
    }
   ],
   "source": [
    "for folder in ['db_agree_dups', 'db_agree_no_dups', 'db_no_agree_dups', 'db_no_agree_no_dups']:\n",
    "    for modal in ['DrugBank', 'ChEMBL', 'NCATS']:\n",
    "        input_folder_path = os.path.join(\"./data/drug_withdrawn\", folder, modal)\n",
    "        output_folder_path = os.path.join(\"./model\", folder, modal)\n",
    "        train = \"t5chem train --data_dir %s --output_dir %s --num_classes %s --pretrain %s --task_type classification --num_epoch %s\" % (input_folder_path, output_folder_path, 2, pre_train_model, 100) \n",
    "\n",
    "        os.system(train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "here we predict and evaluate all the metrics for our t5chem model for all split and test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pre_train_model = './models/pretrain/simple/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n",
      "sh: line 1: t5chem: command not found\n"
     ]
    }
   ],
   "source": [
    "for folder in ['db_agree_dups', 'db_agree_no_dups', 'db_no_agree_dups', 'db_no_agree_no_dups']:\n",
    "    for modal in ['DrugBank', 'ChEMBL', 'NCATS']:\n",
    "        input_folder_path = os.path.join(\"./data/drug_withdrawn\", folder, modal)\n",
    "        model_folder_path = os.path.join(\"./model\", folder, modal)\n",
    "        pred_output_path = os.path.join(\"./model\", folder, modal, 'raw_prob.csv')\n",
    "        predict = 't5chem predict --data_dir %s --model_dir %s --prediction %s' % (input_folder_path, model_folder_path, pred_output_path)\n",
    "        \n",
    "        os.system(predict)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_test(model_path):\n",
    "    df = pd.read_csv(os.path.join(model_path, 'preds.csv'), index_col=0)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import precision_recall_curve, roc_curve, auc, average_precision_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix, matthews_corrcoef, roc_auc_score, average_precision_score, accuracy_score\n",
    "\n",
    "def get_scores(df):\n",
    "    y = np.array(df['target'].tolist())\n",
    "    pred = np.array(df['prob'].tolist())\n",
    "    fpr, tpr, thresholds = roc_curve(y, pred)\n",
    "    precision, recall, thresholds = precision_recall_curve(y, pred)\n",
    "    mcc = matthews_corrcoef(y_true=y, y_pred=(pred > 0.5))\n",
    "    tn, fp, fn, tp = confusion_matrix(y_true=y, y_pred=(pred > 0.5)).ravel()\n",
    "    specificity = tn / (tn+fp)\n",
    "    sensitivity = tp / (tp+fn)\n",
    "    roc = roc_auc_score(y, pred)\n",
    "    aupr = average_precision_score(y, pred)\n",
    "    acc = accuracy_score(y_true=y, y_pred=(pred > 0.5))\n",
    "    return {'auroc': auc(fpr, tpr),'auprc': aupr,'roc': roc,'acc': acc,'mcc': mcc,'sp': specificity,'sn': sensitivity}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "final_scores_df = pd.DataFrame(columns=['auprc', 'auroc', 'roc', 'acc', 'mcc', 'sp', 'sn', 'folder', 'modal'])\n",
    "for folder in ['db_agree_no_dups', 'db_no_agree_dups']:\n",
    "    for modal in ['ChEMBL', 'DrugBank','NCATS']:\n",
    "        input_folder_path = os.path.join(\"./model\", folder, modal)\n",
    "        df = load_test(input_folder_path)\n",
    "        \n",
    "        dic = get_scores(df)\n",
    "        dic.update({'folder': folder, 'modal': modal})\n",
    "        final_scores_df.loc[len(final_scores_df)] = pd.Series(dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auprc</th>\n",
       "      <th>auroc</th>\n",
       "      <th>roc</th>\n",
       "      <th>acc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>sp</th>\n",
       "      <th>sn</th>\n",
       "      <th>folder</th>\n",
       "      <th>modal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.600002</td>\n",
       "      <td>0.778345</td>\n",
       "      <td>0.778345</td>\n",
       "      <td>0.871948</td>\n",
       "      <td>0.479812</td>\n",
       "      <td>0.961219</td>\n",
       "      <td>0.431694</td>\n",
       "      <td>db_agree_no_dups</td>\n",
       "      <td>ChEMBL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.389629</td>\n",
       "      <td>0.804716</td>\n",
       "      <td>0.804716</td>\n",
       "      <td>0.336024</td>\n",
       "      <td>0.128098</td>\n",
       "      <td>0.282489</td>\n",
       "      <td>0.921687</td>\n",
       "      <td>db_agree_no_dups</td>\n",
       "      <td>DrugBank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.42543</td>\n",
       "      <td>0.688401</td>\n",
       "      <td>0.688401</td>\n",
       "      <td>0.677312</td>\n",
       "      <td>0.235812</td>\n",
       "      <td>0.714749</td>\n",
       "      <td>0.551932</td>\n",
       "      <td>db_agree_no_dups</td>\n",
       "      <td>NCATS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.869888</td>\n",
       "      <td>0.923874</td>\n",
       "      <td>0.923874</td>\n",
       "      <td>0.906823</td>\n",
       "      <td>0.734971</td>\n",
       "      <td>0.962032</td>\n",
       "      <td>0.732143</td>\n",
       "      <td>db_no_agree_dups</td>\n",
       "      <td>ChEMBL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.768516</td>\n",
       "      <td>0.910294</td>\n",
       "      <td>0.910294</td>\n",
       "      <td>0.95146</td>\n",
       "      <td>0.719035</td>\n",
       "      <td>0.967902</td>\n",
       "      <td>0.785388</td>\n",
       "      <td>db_no_agree_dups</td>\n",
       "      <td>DrugBank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.54772</td>\n",
       "      <td>0.772386</td>\n",
       "      <td>0.772386</td>\n",
       "      <td>0.705706</td>\n",
       "      <td>0.360215</td>\n",
       "      <td>0.715952</td>\n",
       "      <td>0.678506</td>\n",
       "      <td>db_no_agree_dups</td>\n",
       "      <td>NCATS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      auprc     auroc       roc       acc       mcc        sp        sn  \\\n",
       "0  0.600002  0.778345  0.778345  0.871948  0.479812  0.961219  0.431694   \n",
       "1  0.389629  0.804716  0.804716  0.336024  0.128098  0.282489  0.921687   \n",
       "2   0.42543  0.688401  0.688401  0.677312  0.235812  0.714749  0.551932   \n",
       "3  0.869888  0.923874  0.923874  0.906823  0.734971  0.962032  0.732143   \n",
       "4  0.768516  0.910294  0.910294   0.95146  0.719035  0.967902  0.785388   \n",
       "5   0.54772  0.772386  0.772386  0.705706  0.360215  0.715952  0.678506   \n",
       "\n",
       "             folder     modal  \n",
       "0  db_agree_no_dups    ChEMBL  \n",
       "1  db_agree_no_dups  DrugBank  \n",
       "2  db_agree_no_dups     NCATS  \n",
       "3  db_no_agree_dups    ChEMBL  \n",
       "4  db_no_agree_dups  DrugBank  \n",
       "5  db_no_agree_dups     NCATS  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_scores_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "{'hello': 1}.update({'world':2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import T5ForConditionalGeneration\n",
    "from t5chem import T5ForProperty, SimpleTokenizer\n",
    "import os\n",
    "import torch\n",
    "from tqdm.auto import tqdm\n",
    "sm = torch.nn.Softmax(dim=1)\n",
    "\n",
    "final_scores_df = pd.DataFrame(columns=['auprc', 'auroc', 'roc', 'acc', 'mcc', 'sp', 'sn', 'folder', 'modal'])\n",
    "for folder in ['db_agree_no_dups', 'db_agree_dups']:\n",
    "    for modal in ['ChEMBL', 'DrugBank',' NCATS']:\n",
    "        input_folder_path = os.path.join(\"./data/drug_withdrawn\", folder, modal)\n",
    "        model_folder_path = os.path.join(\"./model\", folder, modal)\n",
    "        tokenizer = SimpleTokenizer(vocab_file=os.path.join(model_folder_path, 'vocab.pt'))\n",
    "        model = T5ForProperty.from_pretrained(model_folder_path)  # for non-seq2seq task\n",
    "        df = load_test(input_folder_path, model_folder_path)\n",
    "        probs = []\n",
    "        for i, row in tqdm(df.iterrows()):\n",
    "            inputs = tokenizer.encode(\"Classification:\" + row['smiles'], return_tensors='pt')\n",
    "            outputs = model(inputs)\n",
    "            probs.append(sm(outputs.logits)[0][-1].detach().numpy().item())\n",
    "        df['prob'] = probs\n",
    "        df.to_csv(os.path.join(model_folder_path, 'preds.csv'))\n",
    "        dic = get_scores(df)\n",
    "        dic.update({'folder': folder, 'modal': modal})\n",
    "        final_scores_df.loc[len(final_scores_df)] = pd.Series(dic)\n",
    "#         final_scores_df = pd.concat([final_scores_df, pd.DataFrame([pd.Series(dic)])], ignore_index=True)\n",
    "#         final_scores_df = final_scores_df.append(pd.Series(dic), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auprc</th>\n",
       "      <th>auroc</th>\n",
       "      <th>folder</th>\n",
       "      <th>modal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.868661</td>\n",
       "      <td>0.938558</td>\n",
       "      <td>db_agree_dups</td>\n",
       "      <td>DrugBank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.876535</td>\n",
       "      <td>0.933469</td>\n",
       "      <td>db_agree_dups</td>\n",
       "      <td>ChEMBL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.530063</td>\n",
       "      <td>0.783790</td>\n",
       "      <td>db_agree_dups</td>\n",
       "      <td>NCATS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.389629</td>\n",
       "      <td>0.804716</td>\n",
       "      <td>db_agree_no_dups</td>\n",
       "      <td>DrugBank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.600002</td>\n",
       "      <td>0.778345</td>\n",
       "      <td>db_agree_no_dups</td>\n",
       "      <td>ChEMBL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.425430</td>\n",
       "      <td>0.688401</td>\n",
       "      <td>db_agree_no_dups</td>\n",
       "      <td>NCATS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.768516</td>\n",
       "      <td>0.910294</td>\n",
       "      <td>db_no_agree_dups</td>\n",
       "      <td>DrugBank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.869888</td>\n",
       "      <td>0.923874</td>\n",
       "      <td>db_no_agree_dups</td>\n",
       "      <td>ChEMBL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.547720</td>\n",
       "      <td>0.772386</td>\n",
       "      <td>db_no_agree_dups</td>\n",
       "      <td>NCATS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.364341</td>\n",
       "      <td>0.771137</td>\n",
       "      <td>db_no_agree_no_dups</td>\n",
       "      <td>DrugBank</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.727618</td>\n",
       "      <td>0.821165</td>\n",
       "      <td>db_no_agree_no_dups</td>\n",
       "      <td>ChEMBL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.507417</td>\n",
       "      <td>0.699560</td>\n",
       "      <td>db_no_agree_no_dups</td>\n",
       "      <td>NCATS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       auprc     auroc               folder     modal\n",
       "0   0.868661  0.938558        db_agree_dups  DrugBank\n",
       "1   0.876535  0.933469        db_agree_dups    ChEMBL\n",
       "2   0.530063  0.783790        db_agree_dups     NCATS\n",
       "3   0.389629  0.804716     db_agree_no_dups  DrugBank\n",
       "4   0.600002  0.778345     db_agree_no_dups    ChEMBL\n",
       "5   0.425430  0.688401     db_agree_no_dups     NCATS\n",
       "6   0.768516  0.910294     db_no_agree_dups  DrugBank\n",
       "7   0.869888  0.923874     db_no_agree_dups    ChEMBL\n",
       "8   0.547720  0.772386     db_no_agree_dups     NCATS\n",
       "9   0.364341  0.771137  db_no_agree_no_dups  DrugBank\n",
       "10  0.727618  0.821165  db_no_agree_no_dups    ChEMBL\n",
       "11  0.507417  0.699560  db_no_agree_no_dups     NCATS"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_scores_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_scores_df.to_csv('./T5chem_final_results.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}