preprocessing/lincs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/icb/yuge.ji/miniconda3/envs/py37/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scanpy==1.6.0 anndata==0.7.4 umap==0.4.6 numpy==1.19.2 scipy==1.6.1 pandas==1.2.3 scikit-learn==0.23.2 statsmodels==0.11.1 python-igraph==0.8.3 leidenalg==0.8.3\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import scanpy as sc\n",
    "sc.set_figure_params(dpi=100, frameon=False)\n",
    "sc.logging.print_header()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/icb/yuge.ji/miniconda3/envs/py37/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.chdir('./../')\n",
    "from cpa.helper import rank_genes_groups_by_cov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata = sc.read('datasets/lincs.h5ad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata.obs['condition'] = adata.obs['pert_iname']\n",
    "adata.obs['cell_type'] = adata.obs['cell_id']\n",
    "adata.obs['dose_val'] = adata.obs['pert_dose']\n",
    "adata.obs['cov_drug_dose_name'] = adata.obs.cell_type.astype(str) + '_' + adata.obs.condition.astype(str) + '_' + adata.obs.dose_val.astype(str)\n",
    "adata.obs['control'] = (adata.obs['condition'] == 'DMSO').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>cell_type</th>\n",
       "      <th>A375</th>\n",
       "      <th>A549</th>\n",
       "      <th>A673</th>\n",
       "      <th>AGS</th>\n",
       "      <th>ASC</th>\n",
       "      <th>ASC.C</th>\n",
       "      <th>BT20</th>\n",
       "      <th>CD34</th>\n",
       "      <th>CL34</th>\n",
       "      <th>CORL23</th>\n",
       "      <th>...</th>\n",
       "      <th>SW620</th>\n",
       "      <th>SW948</th>\n",
       "      <th>T3M10</th>\n",
       "      <th>THP1</th>\n",
       "      <th>TYKNU</th>\n",
       "      <th>U266</th>\n",
       "      <th>U937</th>\n",
       "      <th>VCAP</th>\n",
       "      <th>WSUDLCL2</th>\n",
       "      <th>YAPC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>condition</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1B Parent</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2-methoxyestradiol</th>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3,6-dimethoxyflavone</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3-amino-benzamide</th>\n",
       "      <td>30</td>\n",
       "      <td>61</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5-methoxy-alpha-methyltryptamine</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xanthoxyline</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yohimbine</th>\n",
       "      <td>35</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zacopride</th>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zaprinast</th>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zileuton</th>\n",
       "      <td>32</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1001 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "cell_type                         A375  A549  A673  AGS  ASC  ASC.C  BT20  \\\n",
       "condition                                                                   \n",
       "1B Parent                            8     0     0    0    0      0     0   \n",
       "2-methoxyestradiol                  18     0     0    0    0      0     0   \n",
       "3,6-dimethoxyflavone                 5     3     0    0    3      0     0   \n",
       "3-amino-benzamide                   30    61     2    2    0      0     0   \n",
       "5-methoxy-alpha-methyltryptamine     0     0     0    0    0      0     0   \n",
       "...                                ...   ...   ...  ...  ...    ...   ...   \n",
       "xanthoxyline                         3     5     0    0    4      0     0   \n",
       "yohimbine                           35    16     0    0    4      0     0   \n",
       "zacopride                            9    11     0    0    0      0     0   \n",
       "zaprinast                           21     6     0    0    4      0     0   \n",
       "zileuton                            32    13     2    2    0      0     0   \n",
       "\n",
       "cell_type                         CD34  CL34  CORL23  ...  SW620  SW948  \\\n",
       "condition                                             ...                 \n",
       "1B Parent                            0     0       0  ...      0      0   \n",
       "2-methoxyestradiol                   0     0       0  ...      0      0   \n",
       "3,6-dimethoxyflavone                 0     0       0  ...      0      0   \n",
       "3-amino-benzamide                    0     2       2  ...      4      4   \n",
       "5-methoxy-alpha-methyltryptamine     0     0       0  ...      0      0   \n",
       "...                                ...   ...     ...  ...    ...    ...   \n",
       "xanthoxyline                         0     0       0  ...      0      0   \n",
       "yohimbine                            0     0       0  ...      0      0   \n",
       "zacopride                            0     0       0  ...      0      0   \n",
       "zaprinast                            0     0       0  ...      0      0   \n",
       "zileuton                             0     1       1  ...      3      4   \n",
       "\n",
       "cell_type                         T3M10  THP1  TYKNU  U266  U937  VCAP  \\\n",
       "condition                                                                \n",
       "1B Parent                             0     0      0     0     0     0   \n",
       "2-methoxyestradiol                    0     0      0     0     0     0   \n",
       "3,6-dimethoxyflavone                  0     0      0     0     0     0   \n",
       "3-amino-benzamide                     2     2      2     0     2     9   \n",
       "5-methoxy-alpha-methyltryptamine      0     0      0     0     0     0   \n",
       "...                                 ...   ...    ...   ...   ...   ...   \n",
       "xanthoxyline                          0     0      0     0     0     7   \n",
       "yohimbine                             0     0      0     0     0    26   \n",
       "zacopride                             0     0      0     0     0    19   \n",
       "zaprinast                             0     0      0     0     0    12   \n",
       "zileuton                              2     1      2     0     2    19   \n",
       "\n",
       "cell_type                         WSUDLCL2  YAPC  \n",
       "condition                                         \n",
       "1B Parent                                0     0  \n",
       "2-methoxyestradiol                       0    17  \n",
       "3,6-dimethoxyflavone                     0     0  \n",
       "3-amino-benzamide                        2    18  \n",
       "5-methoxy-alpha-methyltryptamine         0     0  \n",
       "...                                    ...   ...  \n",
       "xanthoxyline                             0     0  \n",
       "yohimbine                                0    18  \n",
       "zacopride                                0     0  \n",
       "zaprinast                                0    18  \n",
       "zileuton                                 2    18  \n",
       "\n",
       "[1001 rows x 82 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(adata.obs.condition, adata.obs.cell_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate differential genes manually, such that the genes are the same per condition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Default of the method has been changed to 't-test' from 't-test_overestim_var'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "... storing 'cov_drug_dose_name' as categorical\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4min 4s, sys: 412 ms, total: 4min 4s\n",
      "Wall time: 4min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sc.tl.rank_genes_groups(\n",
    "    adata,\n",
    "    groupby='condition', \n",
    "    reference='DMSO',\n",
    "    rankby_abs=True,\n",
    "    n_genes=50\n",
    ")\n",
    "\n",
    "de_genes = {}\n",
    "for cond in adata.obs['condition']:\n",
    "    if cond != 'DMSO':\n",
    "        df = sc.get.rank_genes_groups_df(adata, group=cond)  # this takes a while\n",
    "        de_genes[cond] = df['names'][:50].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata.uns['rank_genes_groups_cov'] = {cond: de_genes[cond.split('_')[1]] for cond in adata.obs['cov_drug_dose_name'].unique() if cond.split('_')[1] != 'DMSO'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "adata.obs['split'] = 'train'\n",
    "\n",
    "# take ood from top occurring perturbations to avoid losing data on low occ ones\n",
    "ood_idx = sc.pp.subsample(\n",
    "    adata[adata.obs.condition.isin(list(adata.obs.condition.value_counts().index[1:50]))],\n",
    "    .1,\n",
    "    copy=True\n",
    ").obs.index\n",
    "adata.obs['split'].loc[ood_idx] = 'ood'\n",
    "\n",
    "# take test from a random subsampling of the rest\n",
    "test_idx = sc.pp.subsample(\n",
    "    adata[adata.obs.split != 'ood'],\n",
    "    .16,\n",
    "    copy=True\n",
    ").obs.index\n",
    "adata.obs['split'].loc[test_idx] = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>condition</th>\n",
       "      <th>1B Parent</th>\n",
       "      <th>2-methoxyestradiol</th>\n",
       "      <th>3,6-dimethoxyflavone</th>\n",
       "      <th>3-amino-benzamide</th>\n",
       "      <th>5-methoxy-alpha-methyltryptamine</th>\n",
       "      <th>ABT-737</th>\n",
       "      <th>AG-490</th>\n",
       "      <th>AG-14361</th>\n",
       "      <th>AICA-ribonucleotide</th>\n",
       "      <th>ALW-II-38-3</th>\n",
       "      <th>...</th>\n",
       "      <th>veliparib</th>\n",
       "      <th>vinburnine</th>\n",
       "      <th>voglibose</th>\n",
       "      <th>wiskostatin</th>\n",
       "      <th>xanthohumol</th>\n",
       "      <th>xanthoxyline</th>\n",
       "      <th>yohimbine</th>\n",
       "      <th>zacopride</th>\n",
       "      <th>zaprinast</th>\n",
       "      <th>zileuton</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ood</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>135</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>59</td>\n",
       "      <td>4</td>\n",
       "      <td>125</td>\n",
       "      <td>23</td>\n",
       "      <td>66</td>\n",
       "      <td>50</td>\n",
       "      <td>89</td>\n",
       "      <td>...</td>\n",
       "      <td>188</td>\n",
       "      <td>24</td>\n",
       "      <td>20</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>60</td>\n",
       "      <td>26</td>\n",
       "      <td>33</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>20</td>\n",
       "      <td>100</td>\n",
       "      <td>30</td>\n",
       "      <td>343</td>\n",
       "      <td>14</td>\n",
       "      <td>583</td>\n",
       "      <td>130</td>\n",
       "      <td>272</td>\n",
       "      <td>249</td>\n",
       "      <td>509</td>\n",
       "      <td>...</td>\n",
       "      <td>936</td>\n",
       "      <td>150</td>\n",
       "      <td>105</td>\n",
       "      <td>58</td>\n",
       "      <td>58</td>\n",
       "      <td>53</td>\n",
       "      <td>266</td>\n",
       "      <td>81</td>\n",
       "      <td>175</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 1001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "condition  1B Parent  2-methoxyestradiol  3,6-dimethoxyflavone  \\\n",
       "split                                                            \n",
       "ood                0                   0                     0   \n",
       "test               2                  17                     4   \n",
       "train             20                 100                    30   \n",
       "\n",
       "condition  3-amino-benzamide  5-methoxy-alpha-methyltryptamine  ABT-737  \\\n",
       "split                                                                     \n",
       "ood                        0                                 0       64   \n",
       "test                      59                                 4      125   \n",
       "train                    343                                14      583   \n",
       "\n",
       "condition  AG-490  AG-14361  AICA-ribonucleotide  ALW-II-38-3  ...  veliparib  \\\n",
       "split                                                          ...              \n",
       "ood             0         0                    0            0  ...        135   \n",
       "test           23        66                   50           89  ...        188   \n",
       "train         130       272                  249          509  ...        936   \n",
       "\n",
       "condition  vinburnine  voglibose  wiskostatin  xanthohumol  xanthoxyline  \\\n",
       "split                                                                      \n",
       "ood                 0          0            0            0             0   \n",
       "test               24         20            7            9             9   \n",
       "train             150        105           58           58            53   \n",
       "\n",
       "condition  yohimbine  zacopride  zaprinast  zileuton  \n",
       "split                                                 \n",
       "ood                0          0          0         0  \n",
       "test              60         26         33        51  \n",
       "train            266         81        175       279  \n",
       "\n",
       "[3 rows x 1001 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(adata.obs['split'], adata.obs['condition'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "del(adata.uns['rank_genes_groups'])  # too large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# code compatibility\n",
    "from scipy import sparse\n",
    "adata.X = sparse.csr_matrix(adata.X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "... storing 'split' as categorical\n"
     ]
    }
   ],
   "source": [
    "sc.write('datasets/lincs.h5ad', adata)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}