06_evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit import RDLogger\n",
    "RDLogger.DisableLog('rdApp.*')\n",
    "\n",
    "from evaluation_functions import check_smiles_validity, check_overall_accuracy, check_chirality_weighted_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analyze Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check SMILES validity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>augmentation</th>\n",
       "      <th>top1</th>\n",
       "      <th>top2</th>\n",
       "      <th>top3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>not_augmented</td>\n",
       "      <td>0.997665</td>\n",
       "      <td>0.997802</td>\n",
       "      <td>0.997802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>augmented_2x</td>\n",
       "      <td>0.997383</td>\n",
       "      <td>0.997521</td>\n",
       "      <td>0.997521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>augmented_5x</td>\n",
       "      <td>0.997261</td>\n",
       "      <td>0.997513</td>\n",
       "      <td>0.997513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>augmented_10x</td>\n",
       "      <td>0.997319</td>\n",
       "      <td>0.997474</td>\n",
       "      <td>0.997658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>augmented_20x</td>\n",
       "      <td>0.997249</td>\n",
       "      <td>0.997414</td>\n",
       "      <td>0.997486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>scrambled_5x</td>\n",
       "      <td>0.997317</td>\n",
       "      <td>0.997345</td>\n",
       "      <td>0.997373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    augmentation      top1      top2      top3\n",
       "0  not_augmented  0.997665  0.997802  0.997802\n",
       "1   augmented_2x  0.997383  0.997521  0.997521\n",
       "2   augmented_5x  0.997261  0.997513  0.997513\n",
       "3  augmented_10x  0.997319  0.997474  0.997658\n",
       "4  augmented_20x  0.997249  0.997414  0.997486\n",
       "5   scrambled_5x  0.997317  0.997345  0.997373"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smiles_validity = check_smiles_validity(directories=['not_augmented', 'augmented_2x', 'augmented_5x', 'augmented_10x', 'augmented_20x', 'scrambled_5x'])\n",
    "smiles_validity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check overall accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>augmentation</th>\n",
       "      <th>top1</th>\n",
       "      <th>top2</th>\n",
       "      <th>top3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>not_augmented</td>\n",
       "      <td>0.870</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>augmented_2x</td>\n",
       "      <td>0.862</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>augmented_5x</td>\n",
       "      <td>0.838</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>augmented_10x</td>\n",
       "      <td>0.824</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>augmented_20x</td>\n",
       "      <td>0.830</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>scrambled_5x</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.329</td>\n",
       "      <td>0.370</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    augmentation   top1   top2   top3\n",
       "0  not_augmented  0.870  0.928  0.936\n",
       "1   augmented_2x  0.862  0.959  0.978\n",
       "2   augmented_5x  0.838  0.944  0.968\n",
       "3  augmented_10x  0.824  0.935  0.961\n",
       "4  augmented_20x  0.830  0.943  0.967\n",
       "5   scrambled_5x  0.250  0.329  0.370"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_accuracy = check_overall_accuracy(directories=['not_augmented', 'augmented_2x', 'augmented_5x', 'augmented_10x', 'augmented_20x', 'scrambled_5x'])\n",
    "overall_accuracy = round(overall_accuracy, 3)\n",
    "overall_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check chirality weighted accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>augmentation</th>\n",
       "      <th>top1</th>\n",
       "      <th>top2</th>\n",
       "      <th>top3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>not_augmented</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>augmented_2x</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>augmented_5x</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>augmented_10x</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>augmented_20x</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>scrambled_5x</td>\n",
       "      <td>0.613</td>\n",
       "      <td>0.717</td>\n",
       "      <td>0.753</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    augmentation   top1   top2   top3\n",
       "0  not_augmented  0.952  0.976  0.979\n",
       "1   augmented_2x  0.950  0.987  0.993\n",
       "2   augmented_5x  0.944  0.985  0.991\n",
       "3  augmented_10x  0.942  0.983  0.990\n",
       "4  augmented_20x  0.944  0.984  0.992\n",
       "5   scrambled_5x  0.613  0.717  0.753"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weighted_accuracy = check_chirality_weighted_accuracy(directories=['not_augmented', 'augmented_2x', 'augmented_5x', 'augmented_10x', 'augmented_20x', 'scrambled_5x'])\n",
    "weighted_accuracy = round(weighted_accuracy, 3)\n",
    "weighted_accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chiralpred",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}