LSTM Course Embedder - Seen Classes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from IPython.display import clear_output\n",
    "from pytorch_metric_learning import miners, losses, samplers\n",
    "from pytorch_metric_learning.utils.accuracy_calculator import AccuracyCalculator\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import neptune\n",
    "\n",
    "neptune.init('gregrolwes/course-embedder',)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARAMS = {\n",
    "    'lr': 0.001,\n",
    "    'group size': 8,\n",
    "    'batch size': 32,\n",
    "    'token embedding dim': 256,\n",
    "    'hidden dim': 200,\n",
    "    'output dim': 256,\n",
    "    'num layers': 2,\n",
    "    'bidirectional': True,\n",
    "    'dropout': 0,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neptune.create_experiment(name=\"\", params=PARAMS, tags=['WOS', 'shared embedding space'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import torch\n",
    "from torch import nn\n",
    "from torchtext import data\n",
    "from torchtext.data import TabularDataset, Example\n",
    "\n",
    "# make the experiment reproducible\n",
    "SEED = 42\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = data.Field(include_lengths = True)\n",
    "COURSE = data.LabelField(dtype = torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = [('course', COURSE), ('text', TEXT)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = TabularDataset(\n",
    "    path = 'data/all_WOS.csv',\n",
    "    format = 'csv',\n",
    "    fields = fields,\n",
    ")\n",
    "train_data = TabularDataset(\n",
    "    path = 'data/train_WOS.csv',\n",
    "    format = 'csv',\n",
    "    fields = fields,\n",
    ")\n",
    "test_data = TabularDataset(\n",
    "    path = 'data/test_WOS.csv',\n",
    "    format = 'csv',\n",
    "    fields = fields,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, valid_data = train_data.split(split_ratio=0.9, random_state = random.seed(SEED))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT.build_vocab(train_data, max_size=50000)\n",
    "COURSE.build_vocab(all_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\n",
    "print(f\"Unique tokens in COURSE vocabulary: {len(COURSE.vocab)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TripletIterator(data.Iterator):\n",
    "    def __init__(self, dataset, sampler, **kwargs):\n",
    "        self.sampler = sampler\n",
    "        super(TripletIterator, self).__init__(dataset, shuffle=False, **kwargs)\n",
    "    \n",
    "    def data(self):\n",
    "        \"\"\"Return the examples in the dataset in sampled order.\"\"\"\n",
    "        xs = [self.dataset[i] for i in self.sampler]\n",
    "        return xs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_targets = [COURSE.vocab.stoi[sample.course] for sample in train_data]\n",
    "sampler = samplers.MPerClassSampler(train_targets, 4, length_before_new_iter=len(train_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = PARAMS['batch size']\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "train_iterator = TripletIterator(\n",
    "    train_data,\n",
    "    sampler,\n",
    "    batch_size = BATCH_SIZE,\n",
    "    sort_key = lambda x: len(x.text),\n",
    "    sort_within_batch = True,\n",
    "    train=True,\n",
    "    device = device)\n",
    "\n",
    "valid_iterator = data.Iterator(\n",
    "    valid_data,\n",
    "    batch_size = BATCH_SIZE,\n",
    "    sort_key = lambda x: len(x.text),\n",
    "    sort_within_batch = True,\n",
    "    shuffle=False,\n",
    "    train=False,\n",
    "    device = device)\n",
    "\n",
    "test_iterator = data.Iterator(\n",
    "    test_data,\n",
    "    batch_size = BATCH_SIZE,\n",
    "    sort_key = lambda x: len(x.text),\n",
    "    sort_within_batch = True,\n",
    "    shuffle=False,\n",
    "    train=False,\n",
    "    device = device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, bidirectional, dropout, pad_idx):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)\n",
    "        self.rnn = nn.LSTM(embedding_dim,\n",
    "                          hidden_dim,\n",
    "                          num_layers=n_layers,\n",
    "                          bidirectional=bidirectional,\n",
    "                          dropout=dropout)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n",
    "        self.activation = nn.Sigmoid()\n",
    "    \n",
    "    def forward(self, text, text_lengths):\n",
    "        embedded = self.dropout(self.embedding(text))\n",
    "        #pack sequence\n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths.type(torch.IntTensor))\n",
    "        packed_output, (hidden, cell) = self.rnn(packed_embedded)\n",
    "        #unpack sequence\n",
    "        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))\n",
    "\n",
    "        return self.activation(self.fc(hidden))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIM = len(TEXT.vocab)\n",
    "EMBEDDING_DIM = PARAMS['token embedding dim']\n",
    "HIDDEN_DIM = PARAMS['hidden dim']\n",
    "OUTPUT_DIM = PARAMS['output dim']\n",
    "N_LAYERS = PARAMS['num layers']\n",
    "BIDIRECTIONAL = PARAMS['bidirectional']\n",
    "DROPOUT = PARAMS['dropout']\n",
    "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "\n",
    "model = RNN(INPUT_DIM,\n",
    "           EMBEDDING_DIM,\n",
    "           HIDDEN_DIM,\n",
    "           OUTPUT_DIM,\n",
    "           N_LAYERS,\n",
    "           BIDIRECTIONAL,\n",
    "           DROPOUT,\n",
    "           PAD_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
    "\n",
    "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "\n",
    "print(model.embedding.weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=PARAMS['lr'])\n",
    "miner = miners.TripletMarginMiner(margin=1.0, type_of_triplets=\"all\")\n",
    "#criterion = losses.MultiSimilarityLoss(alpha=2, beta=50, base=0.5)\n",
    "criterion = losses.TripletMarginLoss(margin=0.1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(device)\n",
    "miner = miner.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def progress_bar(current, final, acc, loss, thing):\n",
    "    progress = int(100*(current/final))\n",
    "    remaining = 100-progress\n",
    "    clear_output(wait=True)\n",
    "    \n",
    "    print(thing)\n",
    "    print('|', end='')\n",
    "    print('='*progress, end='')\n",
    "    print('>', end='')\n",
    "    print(' '*remaining, end='')\n",
    "    print('| {}%'.format(progress))\n",
    "    print(\"Loss:\", loss)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_calculator1 = AccuracyCalculator(include = (\"mean_average_precision_at_r\",), k = 1)\n",
    "accuracy_calculator5 = AccuracyCalculator(include = (\"mean_average_precision_at_r\",), k = 5)\n",
    "accuracy_calculator10 = AccuracyCalculator(include = (\"mean_average_precision_at_r\",), k = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_acc(embeddings, labels, queries, q_labels, same):\n",
    "    p_at_1 = accuracy_calculator1.get_accuracy(queries.numpy(), \n",
    "                                                embeddings.numpy(),\n",
    "                                                q_labels.numpy(),\n",
    "                                                labels.numpy(),\n",
    "                                                same)['mean_average_precision_at_r']\n",
    "    torch.cuda.empty_cache()\n",
    "    p_at_5 = accuracy_calculator5.get_accuracy(queries.numpy(), \n",
    "                                                embeddings.numpy(),\n",
    "                                                q_labels.numpy(),\n",
    "                                                labels.numpy(),\n",
    "                                                same)['mean_average_precision_at_r']\n",
    "    torch.cuda.empty_cache()\n",
    "    p_at_10 = accuracy_calculator10.get_accuracy(queries.numpy(), \n",
    "                                                embeddings.numpy(),\n",
    "                                                q_labels.numpy(),\n",
    "                                                labels.numpy(),\n",
    "                                                same)['mean_average_precision_at_r']\n",
    "    return p_at_1, p_at_5, p_at_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    train_embeddings = torch.Tensor([])\n",
    "    train_labels = torch.Tensor([])\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for i, batch in enumerate(iterator):\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        text, text_lengths = batch.text\n",
    "        outputs = model(text, text_lengths).squeeze(1)\n",
    "        triplets = miner(outputs, batch.course)\n",
    "        loss = criterion(outputs, batch.course, triplets)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        \n",
    "        train_embeddings = torch.cat((train_embeddings, outputs.detach().cpu()))\n",
    "        train_labels = torch.cat((train_labels, batch.course.detach().cpu()))\n",
    "        \n",
    "        if i % 1 == 0:\n",
    "            progress_bar(i, len(iterator), epoch_acc / (i+1), epoch_loss / (i+1), \"TRAINING\")\n",
    "        \n",
    "        torch.cuda.empty_cache()\n",
    "        \n",
    "    train_acc, p_at_5, p_at_10 = get_acc(train_embeddings, train_labels, train_embeddings, train_labels, same=True)\n",
    "    neptune.log_metric('Training P@1', train_acc)\n",
    "    neptune.log_metric('Training P@5', p_at_5)\n",
    "    neptune.log_metric('Training P@10', p_at_10)\n",
    "    \n",
    "    embedding_space = map_features(train_embeddings, train_labels, \"Training\")\n",
    "        \n",
    "    return epoch_loss / len(iterator), train_acc, embedding_space, train_embeddings, train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion, train_embeddings, train_labels):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    val_embeddings = torch.Tensor([])\n",
    "    val_labels = torch.Tensor([])\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for i, batch in enumerate(iterator):\n",
    "            text, text_lengths = batch.text            \n",
    "            outputs = model(text, text_lengths).squeeze(1)\n",
    "            triplets = miner(outputs, batch.course)\n",
    "            loss = criterion(outputs, batch.course, triplets)\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            \n",
    "            val_embeddings = torch.cat((val_embeddings, outputs.detach().cpu()))\n",
    "            val_labels = torch.cat((val_labels, batch.course.detach().cpu()))\n",
    "            \n",
    "            if i % 1 == 0:\n",
    "                progress_bar(i, len(iterator), epoch_acc / (i+1), epoch_loss / (i+1), \"VALIDATION\")\n",
    "\n",
    "            torch.cuda.empty_cache()\n",
    "            \n",
    "    \n",
    "    val_acc, p_at_5, p_at_10 = get_acc(train_embeddings, train_labels, val_embeddings, val_labels, same=False)\n",
    "    neptune.log_metric('Validation P@1', val_acc)\n",
    "    neptune.log_metric('Validation P@5', p_at_5)\n",
    "    neptune.log_metric('Validation P@10', p_at_10)\n",
    "    \n",
    "    embedding_space = map_features(val_embeddings, val_labels, \"Validation\")\n",
    "    \n",
    "    return epoch_loss / len(iterator), val_acc, embedding_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_features(outputs, labels, phase):\n",
    "        # create array of column for each feature output\n",
    "        feat_cols = ['feature' + str(i) for i in range(outputs.shape[1])]\n",
    "\n",
    "        # make dataframe of outputs -> labels\n",
    "        df = pd.DataFrame(outputs, columns=feat_cols)\n",
    "        df['y'] = labels\n",
    "        df['labels'] = df['y'].apply(lambda i: str(i))\n",
    "\n",
    "        # clear outputs and labels\n",
    "        outputs, labels = None, None\n",
    "\n",
    "        # creates an array of random indices from size of outputs\n",
    "        np.random.seed(42)\n",
    "        rand_perm = np.random.permutation(df.shape[0])\n",
    "\n",
    "        num_examples = 10000\n",
    "\n",
    "        df_subset = df.loc[rand_perm[:num_examples], :].copy()\n",
    "        data_subset = df_subset[feat_cols].values\n",
    "\n",
    "        pca = PCA(n_components=50)\n",
    "        pca_result = pca.fit_transform(data_subset)\n",
    "\n",
    "        tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)\n",
    "        tsne_results = tsne.fit_transform(data_subset)\n",
    "        df_subset['tsne-2d-one'] = tsne_results[:, 0]\n",
    "        df_subset['tsne-2d-two'] = tsne_results[:, 1]\n",
    "\n",
    "        plt.figure(figsize=(16, 10))\n",
    "        plt.scatter(\n",
    "            x=df_subset[\"tsne-2d-one\"],\n",
    "            y=df_subset[\"tsne-2d-two\"],\n",
    "            c=df_subset[\"y\"],\n",
    "            s=4\n",
    "        )\n",
    "        plt.axis('off')\n",
    "        plt.title(\"{} Embedding Space\".format(phase))\n",
    "        return plt.gcf()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_EPOCHS = 50\n",
    "\n",
    "# initialize best loss as infinity\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    train_loss, train_acc, embedding_space, emb, labels = train(model, train_iterator, optimizer, criterion)\n",
    "    neptune.log_metric('Training Loss', train_loss)\n",
    "    neptune.log_image('Training Embedding Space', embedding_space, image_name='Epoch {}'.format(epoch+1))\n",
    "    \n",
    "    valid_loss, valid_acc, embedding_space = evaluate(model, valid_iterator, criterion, emb, labels)\n",
    "    neptune.log_metric('Validation Loss', valid_loss)\n",
    "    neptune.log_image('Validation Embedding Space', embedding_space, image_name='Epoch {}'.format(epoch+1))\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'models/course_embedder.pt')\n",
    "    \n",
    "    print(f'Epoch: {epoch+1}')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f}\\t|\\tTrain Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f}\\t|\\tVal. Acc: {valid_acc*100:.2f}%')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loss, test_acc = evaluate(model, test_iterator, criterion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_acc"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "neptune": {
   "notebookId": "4dddba6c-fb27-4a70-9549-df778ed2b7fc"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}