extrinsic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1653a48e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "import time\n",
    "import random\n",
    "import torch\n",
    "\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "from src.model import *\n",
    "from src.load_data import load_data\n",
    "\n",
    "from pathlib import Path\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "from torchtext.vocab import Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3b3e3d8-e3cb-4c0f-895e-4589eec2370e",
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 1234\n",
    "MIN_FREQ = 1\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0b3aec6",
   "metadata": {},
   "outputs": [],
   "source": [
    "windows = [2, 5, 10]\n",
    "vector_sizes = [0, 50, 100, 500]\n",
    "languages = ['bxr']#['bxr', 'myv', 'kv']#['bxr', 'myv', 'kv']\n",
    "methods = ['cbow', 'sg', 'glove']#[-1, 'cbow', 'sg', 'glove', 'pmi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe139062",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_d = {}\n",
    "for language in languages:\n",
    "    data_generator = load_data(language = language, SEED = SEED)\n",
    "    \n",
    "    scores_d[language] = {}\n",
    "    for method in methods:\n",
    "        if method not in scores_d[language]:\n",
    "            scores_d[language][method] = {}\n",
    "        \n",
    "        for vector_size in vector_sizes:\n",
    "            if vector_size not in scores_d[language][method]:\n",
    "                scores_d[language][method][vector_size] = {}\n",
    "                    \n",
    "            for window in windows:            \n",
    "                if window not in scores_d[language][method][vector_size]:\n",
    "                    scores_d[language][method][vector_size][window] = []\n",
    "                \n",
    "                print(\"language:\", language, \"method:\", method, \"vector_size:\", vector_size, \"window:\", window)\n",
    "                if method != -1:\n",
    "                    path = Path(\"./embeddings/\"+language+\"/\"+str(vector_size)+\"/\"+str(window)+\"/\"+method)\n",
    "                    if not path.is_file():\n",
    "                        print(\"File\", path, \"doesn't exist\")\n",
    "                        continue\n",
    "                    else:\n",
    "                        print(\"loaded word embeddings\", path)\n",
    "                else:\n",
    "                    if vector_size == 0:\n",
    "                        print(\"skipping\")\n",
    "                        continue\n",
    "                \n",
    "                for NUM, TEXT, LEMMA, UD_TAGS, train_data, val_data in data_generator.get_fold_data():\n",
    "                    \n",
    "                    print(\"train_data\", len(train_data.examples), \"val_data\", len(val_data.examples))\n",
    "                    \n",
    "                    if method == -1:\n",
    "                        TEXT.build_vocab(train_data, min_freq = MIN_FREQ)\n",
    "                        emb_size = vector_size\n",
    "                    else:\n",
    "                        !rm -rf /tmp/vec\n",
    "                        _vectors = Vectors(name=path, cache='/tmp/vec')\n",
    "                        TEXT.build_vocab(train_data,\n",
    "                            min_freq = MIN_FREQ,\n",
    "                            vectors = _vectors,\n",
    "                            unk_init = torch.Tensor.normal_\n",
    "                        )\n",
    "                        emb_size = TEXT.vocab.vectors[1].shape[0]\n",
    "                        \n",
    "                        if vector_size !=0:\n",
    "                            assert vector_size==TEXT.vocab.vectors[1].shape[0], \"Different sizes\"\n",
    "                        \n",
    "                    LEMMA.build_vocab(train_data)\n",
    "                    NUM.build_vocab(train_data)\n",
    "                    UD_TAGS.build_vocab(train_data)\n",
    "                    \n",
    "                    print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\n",
    "                    print(f\"Unique tokens in UD_TAG vocabulary: {len(UD_TAGS.vocab)}\")\n",
    "                    \n",
    "                    train_iterator, val_iterator = data.BucketIterator.splits(\n",
    "                                                        (train_data, val_data),\n",
    "                                                        sort=True,\n",
    "                                                        sort_key=lambda x: len(x.text),\n",
    "                                                        sort_within_batch=False,\n",
    "                                                        batch_size=BATCH_SIZE, \n",
    "                                                        repeat=False,\n",
    "                                                        shuffle=True,\n",
    "                                                        device=device)\n",
    "                    \n",
    "                    pos_score = pos(TEXT, UD_TAGS, train_iterator, val_iterator, emb_size, epochs=30, verbose = False)\n",
    "                    print(\"score:\", pos_score)\n",
    "                    scores_d[language][method][vector_size][window].append(pos_score)\n",
    "                scores_d[language][method][vector_size][window] = np.average(scores_d[language][method][vector_size][window])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afea1779-d485-413c-bf26-833f13c530f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "for idx, vec in enumerate(TEXT.vocab.vectors):\n",
    "    if vec[0] == 0 and not TEXT.vocab.itos[idx].isdigit():\n",
    "#         print(TEXT.vocab.itos[idx])\n",
    "        count += 1\n",
    "\n",
    "print('словарь', len(TEXT.vocab.vectors))\n",
    "print('не найдено', count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "223f321d",
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT.vocab.stoi[TEXT.pad_token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbfa7dda",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pos(TEXT, UD_TAGS, train_iterator, val_iterator, emb_size, epochs=10, verbose = True):\n",
    "      \n",
    "    INPUT_DIM = len(TEXT.vocab)\n",
    "    EMBEDDING_DIM = emb_size\n",
    "    HIDDEN_DIM = 128\n",
    "    OUTPUT_DIM = len(UD_TAGS.vocab)\n",
    "    N_LAYERS = 1\n",
    "    BIDIRECTIONAL = False\n",
    "    DROPOUT = 0.25\n",
    "    PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "\n",
    "    model = BiLSTMPOSTagger(INPUT_DIM, \n",
    "                        EMBEDDING_DIM, \n",
    "                        HIDDEN_DIM, \n",
    "                        OUTPUT_DIM, \n",
    "                        N_LAYERS, \n",
    "                        BIDIRECTIONAL, \n",
    "                        DROPOUT,\n",
    "                        PAD_IDX)\n",
    "    \n",
    "    model.apply(init_weights)\n",
    "    print(f'The model has {count_parameters(model):,} trainable parameters')\n",
    "    \n",
    "    if TEXT.vocab.vectors is not None:\n",
    "        print(\"load pretrained embs\")\n",
    "        pretrained_embeddings = TEXT.vocab.vectors\n",
    "        print('embeddings', pretrained_embeddings.shape)\n",
    "    \n",
    "        model.embedding.weight.data.copy_(pretrained_embeddings)\n",
    "        model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "    else:\n",
    "        print(\"no loading any pretrained embs\")\n",
    "    \n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "    TAG_PAD_IDX = UD_TAGS.vocab.stoi[UD_TAGS.pad_token]\n",
    "    criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n",
    "    \n",
    "    model = model.to(device)\n",
    "    criterion = criterion.to(device)\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "\n",
    "        start_time = time.time()\n",
    "    \n",
    "        train_loss = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n",
    "        valid_loss, valid_acc = evaluate(model, val_iterator, criterion, TAG_PAD_IDX)\n",
    "    \n",
    "        end_time = time.time()\n",
    "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "            \n",
    "        if verbose:\n",
    "            print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "            print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')\n",
    "\n",
    "    test_loss, test_acc = evaluate(model, val_iterator, criterion, TAG_PAD_IDX)\n",
    "    return test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64a8ed6f-1ef6-45c5-bd33-88f3921762cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pprint\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "pp.pprint(scores_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84efef57-01e6-4144-8062-344788244866",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('extrinsic.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(scores_d, f, sort_keys=False, indent=4, ensure_ascii=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}