a-neural-probabilistic-language-model.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T11:21:28.365793Z","iopub.status.busy":"2024-07-13T11:21:28.365354Z","iopub.status.idle":"2024-07-13T11:21:35.649790Z","shell.execute_reply":"2024-07-13T11:21:35.648574Z","shell.execute_reply.started":"2024-07-13T11:21:28.365756Z"},"trusted":true},"outputs":[{"data":{"text/plain":["device(type='cpu')"]},"execution_count":1,"metadata":{},"output_type":"execute_result"}],"source":["import numpy as np \n","import pandas as pd \n","import torch\n","import torch.nn.functional as F\n","import re\n","\n","from nltk.tokenize import word_tokenize\n","from collections import Counter\n","from torch.utils.data import Dataset, DataLoader\n","from torch import nn, optim\n","from tqdm import tqdm\n","from tqdm.auto import tqdm\n","\n","device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","device"]},{"cell_type":"code","execution_count":3,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:32.817355Z","iopub.status.busy":"2024-07-13T06:53:32.816925Z","iopub.status.idle":"2024-07-13T06:53:32.822155Z","shell.execute_reply":"2024-07-13T06:53:32.821302Z","shell.execute_reply.started":"2024-07-13T06:53:32.817328Z"},"trusted":true},"outputs":[],"source":["def alphanumeric_cleansing(text):\n","    # Remove non-alphanumeric characters using regular expression\n","    cleaned_text = re.sub(r'[^a-zA-Z0-9\\s]', '', text)\n","    return cleaned_text"]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:32.825122Z","iopub.status.busy":"2024-07-13T06:53:32.824645Z","iopub.status.idle":"2024-07-13T06:53:45.885807Z","shell.execute_reply":"2024-07-13T06:53:45.884844Z","shell.execute_reply.started":"2024-07-13T06:53:32.825088Z"},"trusted":true},"outputs":[],"source":["# Alphanumeric Cleansing\n","df = pd.read_csv('./brown.csv')\n","df['tokenized_text'] = df['tokenized_text'].apply(lambda row: alphanumeric_cleansing(row))\n","df['tokens'] = df['tokenized_text'].apply(word_tokenize)"]},{"cell_type":"code","execution_count":5,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:45.887106Z","iopub.status.busy":"2024-07-13T06:53:45.886833Z","iopub.status.idle":"2024-07-13T06:53:46.638146Z","shell.execute_reply":"2024-07-13T06:53:46.637203Z","shell.execute_reply.started":"2024-07-13T06:53:45.887082Z"},"trusted":true},"outputs":[],"source":["# Merge rare words\n","all_tokens = [token for sublist in df['tokens'] for token in sublist]\n","token_freq = Counter(all_tokens)\n","df['tokens'] = df['tokens'].apply(lambda tokens: ['UNKNOWN' if token_freq[token]<=3 else token for token in tokens])\n","df['processed_text'] = df['tokens'].apply(lambda tokens: ' '.join(tokens))"]},{"cell_type":"code","execution_count":6,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:46.639579Z","iopub.status.busy":"2024-07-13T06:53:46.639283Z","iopub.status.idle":"2024-07-13T06:53:46.666737Z","shell.execute_reply":"2024-07-13T06:53:46.665796Z","shell.execute_reply.started":"2024-07-13T06:53:46.639554Z"},"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>filename</th>\n","      <th>para_id</th>\n","      <th>sent_id</th>\n","      <th>raw_text</th>\n","      <th>tokenized_text</th>\n","      <th>tokenized_pos</th>\n","      <th>label</th>\n","      <th>tokens</th>\n","      <th>processed_text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>cd05</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>Furthermore/rb ,/, as/cs an/at encouragement/n...</td>\n","      <td>Furthermore  as an encouragement to revisionis...</td>\n","      <td>rb , cs at nn in nn nn , pps rb bez jj to vb c...</td>\n","      <td>religion</td>\n","      <td>[Furthermore, as, an, encouragement, to, UNKNO...</td>\n","      <td>Furthermore as an encouragement to UNKNOWN thi...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>cd05</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>The/at Unitarian/jj clergy/nns were/bed an/at ...</td>\n","      <td>The Unitarian clergy were an exclusive club of...</td>\n","      <td>at jj nns bed at jj nn in vbn nns -- cs at nn ...</td>\n","      <td>religion</td>\n","      <td>[The, Unitarian, clergy, were, an, exclusive, ...</td>\n","      <td>The Unitarian clergy were an exclusive club of...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>cd05</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>Ezra/np Stiles/np Gannett/np ,/, an/at honorab...</td>\n","      <td>Ezra Stiles Gannett  an honorable representati...</td>\n","      <td>np np np , at jj nn in at nn , vbd ppl rb in a...</td>\n","      <td>religion</td>\n","      <td>[UNKNOWN, UNKNOWN, UNKNOWN, an, honorable, rep...</td>\n","      <td>UNKNOWN UNKNOWN UNKNOWN an honorable represent...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>cd05</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>Even/rb so/rb ,/, Gannett/np judiciously/rb ar...</td>\n","      <td>Even so  Gannett judiciously argued  the Assoc...</td>\n","      <td>rb rb , np rb vbd , at nn-tl md rb vb cs np ``...</td>\n","      <td>religion</td>\n","      <td>[Even, so, UNKNOWN, UNKNOWN, argued, the, Asso...</td>\n","      <td>Even so UNKNOWN UNKNOWN argued the Association...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>cd05</td>\n","      <td>0</td>\n","      <td>4</td>\n","      <td>We/ppss today/nr are/ber not/* entitled/vbn to...</td>\n","      <td>We today are not entitled to excoriate honest ...</td>\n","      <td>ppss nr ber * vbn to vb jj nns wps vbd np to b...</td>\n","      <td>religion</td>\n","      <td>[We, today, are, not, entitled, to, UNKNOWN, h...</td>\n","      <td>We today are not entitled to UNKNOWN honest me...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>57335</th>\n","      <td>cj14</td>\n","      <td>6</td>\n","      <td>3</td>\n","      <td>For/in the/at most/ap part/nn ,/, this/dt disc...</td>\n","      <td>For the most part  this discussion will be con...</td>\n","      <td>in at ap nn , dt nn md be vbn in nns vbn in at...</td>\n","      <td>learned</td>\n","      <td>[For, the, most, part, this, discussion, will,...</td>\n","      <td>For the most part this discussion will be conf...</td>\n","    </tr>\n","    <tr>\n","      <th>57336</th>\n","      <td>cj14</td>\n","      <td>6</td>\n","      <td>4</td>\n","      <td>A/np-hl ./.-hl</td>\n","      <td>A</td>\n","      <td>np-hl .-hl</td>\n","      <td>learned</td>\n","      <td>[A]</td>\n","      <td>A</td>\n","    </tr>\n","    <tr>\n","      <th>57337</th>\n","      <td>cj14</td>\n","      <td>6</td>\n","      <td>5</td>\n","      <td>Standard/jj-hl preparations/nns-hl and/cc-hl u...</td>\n","      <td>Standard preparations and units of thyroidstim...</td>\n","      <td>jj-hl nns-hl cc-hl nns-hl in-hl jj-hl nn-hl</td>\n","      <td>learned</td>\n","      <td>[Standard, preparations, and, units, of, thyro...</td>\n","      <td>Standard preparations and units of thyroidstim...</td>\n","    </tr>\n","    <tr>\n","      <th>57338</th>\n","      <td>cj14</td>\n","      <td>6</td>\n","      <td>6</td>\n","      <td>The/at international/jj unit/nn (/( u./nn )/) ...</td>\n","      <td>The international unit  u   adopted to make po...</td>\n","      <td>at jj nn ( nn ) , vbn to vb jj at nn in nns in...</td>\n","      <td>learned</td>\n","      <td>[The, international, unit, UNKNOWN, adopted, t...</td>\n","      <td>The international unit UNKNOWN adopted to make...</td>\n","    </tr>\n","    <tr>\n","      <th>57339</th>\n","      <td>cj14</td>\n","      <td>6</td>\n","      <td>7</td>\n","      <td>The/at international/jj unit/nn is/bez equipot...</td>\n","      <td>The international unit is equipotent with the ...</td>\n","      <td>at jj nn bez jj in at nn nn vbn in cd , wdt be...</td>\n","      <td>learned</td>\n","      <td>[The, international, unit, is, UNKNOWN, with, ...</td>\n","      <td>The international unit is UNKNOWN with the UNK...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>57340 rows × 9 columns</p>\n","</div>"],"text/plain":["      filename  para_id  sent_id  \\\n","0         cd05        0        0   \n","1         cd05        0        1   \n","2         cd05        0        2   \n","3         cd05        0        3   \n","4         cd05        0        4   \n","...        ...      ...      ...   \n","57335     cj14        6        3   \n","57336     cj14        6        4   \n","57337     cj14        6        5   \n","57338     cj14        6        6   \n","57339     cj14        6        7   \n","\n","                                                raw_text  \\\n","0      Furthermore/rb ,/, as/cs an/at encouragement/n...   \n","1      The/at Unitarian/jj clergy/nns were/bed an/at ...   \n","2      Ezra/np Stiles/np Gannett/np ,/, an/at honorab...   \n","3      Even/rb so/rb ,/, Gannett/np judiciously/rb ar...   \n","4      We/ppss today/nr are/ber not/* entitled/vbn to...   \n","...                                                  ...   \n","57335  For/in the/at most/ap part/nn ,/, this/dt disc...   \n","57336                                     A/np-hl ./.-hl   \n","57337  Standard/jj-hl preparations/nns-hl and/cc-hl u...   \n","57338  The/at international/jj unit/nn (/( u./nn )/) ...   \n","57339  The/at international/jj unit/nn is/bez equipot...   \n","\n","                                          tokenized_text  \\\n","0      Furthermore  as an encouragement to revisionis...   \n","1      The Unitarian clergy were an exclusive club of...   \n","2      Ezra Stiles Gannett  an honorable representati...   \n","3      Even so  Gannett judiciously argued  the Assoc...   \n","4      We today are not entitled to excoriate honest ...   \n","...                                                  ...   \n","57335  For the most part  this discussion will be con...   \n","57336                                                 A    \n","57337  Standard preparations and units of thyroidstim...   \n","57338  The international unit  u   adopted to make po...   \n","57339  The international unit is equipotent with the ...   \n","\n","                                           tokenized_pos     label  \\\n","0      rb , cs at nn in nn nn , pps rb bez jj to vb c...  religion   \n","1      at jj nns bed at jj nn in vbn nns -- cs at nn ...  religion   \n","2      np np np , at jj nn in at nn , vbd ppl rb in a...  religion   \n","3      rb rb , np rb vbd , at nn-tl md rb vb cs np ``...  religion   \n","4      ppss nr ber * vbn to vb jj nns wps vbd np to b...  religion   \n","...                                                  ...       ...   \n","57335  in at ap nn , dt nn md be vbn in nns vbn in at...   learned   \n","57336                                         np-hl .-hl   learned   \n","57337        jj-hl nns-hl cc-hl nns-hl in-hl jj-hl nn-hl   learned   \n","57338  at jj nn ( nn ) , vbn to vb jj at nn in nns in...   learned   \n","57339  at jj nn bez jj in at nn nn vbn in cd , wdt be...   learned   \n","\n","                                                  tokens  \\\n","0      [Furthermore, as, an, encouragement, to, UNKNO...   \n","1      [The, Unitarian, clergy, were, an, exclusive, ...   \n","2      [UNKNOWN, UNKNOWN, UNKNOWN, an, honorable, rep...   \n","3      [Even, so, UNKNOWN, UNKNOWN, argued, the, Asso...   \n","4      [We, today, are, not, entitled, to, UNKNOWN, h...   \n","...                                                  ...   \n","57335  [For, the, most, part, this, discussion, will,...   \n","57336                                                [A]   \n","57337  [Standard, preparations, and, units, of, thyro...   \n","57338  [The, international, unit, UNKNOWN, adopted, t...   \n","57339  [The, international, unit, is, UNKNOWN, with, ...   \n","\n","                                          processed_text  \n","0      Furthermore as an encouragement to UNKNOWN thi...  \n","1      The Unitarian clergy were an exclusive club of...  \n","2      UNKNOWN UNKNOWN UNKNOWN an honorable represent...  \n","3      Even so UNKNOWN UNKNOWN argued the Association...  \n","4      We today are not entitled to UNKNOWN honest me...  \n","...                                                  ...  \n","57335  For the most part this discussion will be conf...  \n","57336                                                  A  \n","57337  Standard preparations and units of thyroidstim...  \n","57338  The international unit UNKNOWN adopted to make...  \n","57339  The international unit is UNKNOWN with the UNK...  \n","\n","[57340 rows x 9 columns]"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["df"]},{"cell_type":"code","execution_count":7,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:46.668151Z","iopub.status.busy":"2024-07-13T06:53:46.667882Z","iopub.status.idle":"2024-07-13T06:53:46.771228Z","shell.execute_reply":"2024-07-13T06:53:46.770372Z","shell.execute_reply.started":"2024-07-13T06:53:46.668121Z"},"trusted":true},"outputs":[],"source":["tokens = {token for sublist in df['tokens'] for token in sublist}\n","sorted_tokens = sorted(tokens)\n","vocab = {token: idx for idx, token in enumerate(sorted_tokens)}\n","token_to_word = {v: k for k, v in vocab.items()}"]},{"cell_type":"code","execution_count":8,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:46.772707Z","iopub.status.busy":"2024-07-13T06:53:46.772398Z","iopub.status.idle":"2024-07-13T06:53:50.019230Z","shell.execute_reply":"2024-07-13T06:53:50.018287Z","shell.execute_reply.started":"2024-07-13T06:53:46.772682Z"},"trusted":true},"outputs":[],"source":["counter = 0\n","flag = 0\n","train_sep = -1\n","val_sep = -1\n","for index, row in df.iterrows():\n","    counter += len(row.tokens)\n","    if counter >= 800000 and flag == 0:\n","        train_sep = index\n","        counter = 0\n","        flag = 1\n","        continue\n","    if counter >= 200000 and flag == 1:\n","        val_sep = index\n","        break\n","\n","train_df = df.iloc[:train_sep] \n","val_df = df.iloc[train_sep+1:val_sep]\n","test_df = df.iloc[val_sep+1:]"]},{"cell_type":"code","execution_count":9,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:50.020759Z","iopub.status.busy":"2024-07-13T06:53:50.020463Z","iopub.status.idle":"2024-07-13T06:53:50.032436Z","shell.execute_reply":"2024-07-13T06:53:50.031416Z","shell.execute_reply.started":"2024-07-13T06:53:50.020735Z"},"trusted":true},"outputs":[],"source":["# A short note here that n in the paper is not the same as the n-gram. But instead n_gram = n - 1\n","# So given n = 5, the prediction will use 4-gram instead.\n","\n","def generate_n_gram(text, n_gram):\n","        words = word_tokenize(text)\n","        n_grams = zip(*[words[i:] for i in range(n_gram)])\n","        n_grams = [\" \".join(i) for i in n_grams]\n","        if n_grams:\n","            n_grams.pop()\n","        n_grams = [(n_grams[i], words[i+n_gram]) for i in range(len(n_grams))]\n","        return list(n_grams)\n","    \n","class CustomTextDataset(Dataset):\n","    def __init__(self, n, df, vocab):\n","        self.n = n\n","        self.n_gram = n - 1\n","        self.vocab = vocab\n","        self.V = len(vocab)\n","        temp_df = df['processed_text'].apply(lambda row: generate_n_gram(row, self.n_gram))\n","        self.data = []\n","        for index, row in temp_df.items():\n","            if row:\n","                self.data += row \n","    \n","    def __len__(self):\n","        return len(self.data)\n","    \n","    def __getitem__(self, idx):\n","        ngram, next_word = self.data[idx]\n","        ngram_indices = [self.vocab[word] for word in word_tokenize(ngram)]\n","        next_word_index = self.vocab[next_word]\n","        ngram_indices_tensor = torch.tensor(ngram_indices).to(device)\n","        next_word_index_tensor = torch.tensor(next_word_index).to(device)\n","        return ngram_indices_tensor, next_word_index_tensor"]},{"cell_type":"code","execution_count":10,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:53:50.035607Z","iopub.status.busy":"2024-07-13T06:53:50.035245Z","iopub.status.idle":"2024-07-13T06:54:03.026712Z","shell.execute_reply":"2024-07-13T06:54:03.025717Z","shell.execute_reply.started":"2024-07-13T06:53:50.035576Z"},"trusted":true},"outputs":[],"source":["train_set = CustomTextDataset(5, train_df, vocab)\n","val_set = CustomTextDataset(5, val_df, vocab)\n","test_set = CustomTextDataset(5, test_df,vocab)\n","\n","trainloader = DataLoader(train_set, batch_size=4096)\n","valloader = DataLoader(val_set, batch_size=4096)\n","testloader = DataLoader(test_set, batch_size=4096)"]},{"cell_type":"code","execution_count":13,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:54:04.736550Z","iopub.status.busy":"2024-07-13T06:54:04.736235Z","iopub.status.idle":"2024-07-13T06:54:04.745758Z","shell.execute_reply":"2024-07-13T06:54:04.744695Z","shell.execute_reply.started":"2024-07-13T06:54:04.736525Z"},"trusted":true},"outputs":[],"source":["class NeuralNetLM(nn.Module):\n","    def __init__(self, vocab, m, n, h):\n","        super().__init__()\n","        self.vocab = vocab\n","        self.h = h\n","        self.n = n\n","        self.m = m\n","        self.V = len(vocab)\n","        self.C = nn.Embedding(self.V, self.m)\n","        self.H = nn.Parameter(torch.randn(self.h, (self.n-1) * self.m))\n","        self.d = nn.Parameter(torch.randn(self.h, 1))\n","        self.U = nn.Parameter(torch.randn(self.V, self.h))\n","        self.W = nn.Parameter(torch.randn(self.V, (self.n-1) * self.m))\n","        self.b = nn.Parameter(torch.randn(self.V, 1))\n","    \n","    # Batch forward\n","    # x has a shape of (batch_size, (n-1))\n","    def forward(self, x):\n","        x = self.C(x)\n","        x = x.reshape(self.m * (self.n - 1), -1) # [m*(n-1), batch_size]\n","        x = self.b + self.W @ x + self.U @ torch.tanh((self.H @ x) + self.d)\n","        x = x.T\n","        x = F.log_softmax(x, dim=1)\n","        return x"]},{"cell_type":"code","execution_count":14,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:54:04.747102Z","iopub.status.busy":"2024-07-13T06:54:04.746847Z","iopub.status.idle":"2024-07-13T06:54:06.036082Z","shell.execute_reply":"2024-07-13T06:54:06.035094Z","shell.execute_reply.started":"2024-07-13T06:54:04.747079Z"},"trusted":true},"outputs":[],"source":["model = NeuralNetLM(vocab, m = 60, n = 5, h = 50).to(device)\n","criterion = nn.NLLLoss() # Training is actually done with 2 loss functions, weights use L2, \n","                         # Biases use max. likelihood without regularization term.\n","optimizer = optim.AdamW(model.parameters())"]},{"cell_type":"code","execution_count":15,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T06:54:06.037630Z","iopub.status.busy":"2024-07-13T06:54:06.037192Z","iopub.status.idle":"2024-07-13T06:54:06.044830Z","shell.execute_reply":"2024-07-13T06:54:06.043930Z","shell.execute_reply.started":"2024-07-13T06:54:06.037604Z"},"trusted":true},"outputs":[],"source":["\n","\n","def loop_fn(mode, dataset, dataloader, model, criterion, optimizer, device):\n","    if mode=='train':\n","        model.train()\n","    elif mode=='test':\n","        model.eval()\n","    cost = 0\n","    for feature, target in tqdm(dataloader, desc=mode.title()):\n","        feature, target = feature.to(device), target.to(device)\n","        output = model(feature)\n","        loss = criterion(output, target)\n","        \n","        if mode=='train':\n","            loss.backward()\n","            optimizer.step()\n","            optimizer.zero_grad()\n","        \n","        cost += loss.item() * feature.shape[0]\n","    cost = cost / len(dataset)\n","    return cost"]},{"cell_type":"code","execution_count":19,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T07:04:32.483045Z","iopub.status.busy":"2024-07-13T07:04:32.482683Z","iopub.status.idle":"2024-07-13T07:45:04.583904Z","shell.execute_reply":"2024-07-13T07:45:04.582922Z","shell.execute_reply.started":"2024-07-13T07:04:32.483018Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"37f414991ba84ac88249cceee2517d7c","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"572d000befcc4dc8bb10451d42dbab4b","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 1/20 | train_cost:  38.354642 | test_cost:  39.591419 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"fb7192457c094849be008c1300e0eea5","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1ace16c515e44620a2b61b7a9a53c0c9","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 2/20 | train_cost:  34.611660 | test_cost:  36.478985 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"4cfca7076baf4ef2b16adbf2c23be52a","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"06f8091fd5a44856b781503cab279379","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 3/20 | train_cost:  31.471597 | test_cost:  33.785180 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"dfd3c7b612b941a886b37c6ebb07eef4","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"16bbb6466b8748d6a17aa693d944903f","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 4/20 | train_cost:  28.761548 | test_cost:  31.430763 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"38158e169132443e8339c4120c26e556","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"452003a3e0df44c5af46bdd8ff2b6c1a","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 5/20 | train_cost:  26.403473 | test_cost:  29.334626 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"cb0b90e0bc174d55bee7823aa4695a26","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"72e47b4e67ac4a1eb2ebd2653a653872","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 6/20 | train_cost:  24.341675 | test_cost:  27.490616 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"94ef48d439454c099587f050d3bdb660","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2789a6dca5cd47be8c9ef31fcd36f6d1","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 7/20 | train_cost:  22.530679 | test_cost:  25.865943 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c0c52380cd4a4191940717398fe4ee79","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"e86393e09a1642a6bc11139650eadc73","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 8/20 | train_cost:  20.938989 | test_cost:  24.452043 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c92324abae7842e3a1ad803ce19b2e26","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b14e8c6ad8b843e5853ff88225eeb3ee","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 9/20 | train_cost:  19.535496 | test_cost:  23.203960 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"4e4e197da5e44cf0a038f230da33c4e8","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"88e1eae12cb841b69a9a7d1d4bc66e90","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 10/20 | train_cost:  18.294440 | test_cost:  22.095610 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"cecdd65eaa5b419da45f69761d5f5417","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0cfb4217903741f8a2f3f9d2fbbf45fd","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 11/20 | train_cost:  17.193149 | test_cost:  21.112482 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"5cbdbe72eba145b08433acbf510791b6","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f6bb371a6a264d53aa2eb2ff3f43b346","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 12/20 | train_cost:  16.210646 | test_cost:  20.237766 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0a8f25f635544186b508abbd37382feb","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d29639a94b75458292884a60bf0335c6","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 13/20 | train_cost:  15.331110 | test_cost:  19.453096 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"58ac9d5ddc5f4f36864fe2645d5f43e3","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6c81ab7e10d64403b1e9fdbc3b3dcfb3","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 14/20 | train_cost:  14.540216 | test_cost:  18.747457 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f19ec3a242ae4bb4a4ea02b91bb72431","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"faf1f230f5f9460fb914c4a4cf2cfdd2","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 15/20 | train_cost:  13.825717 | test_cost:  18.109328 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"2da2107259f04ead8c6187eeba965c9e","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1413cd4cfcc34e61bb7584f2071b6f08","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 16/20 | train_cost:  13.177628 | test_cost:  17.529509 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"70e89820fd04476e8deedbcafe200c85","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b932a4c1fb8e4208884c9b81e0488d4b","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 17/20 | train_cost:  12.587641 | test_cost:  17.000324 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"09a585440b4e4a0cb3411c09d92e8a60","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9c82a25284a641d08da9dfb803ee8b2c","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 18/20 | train_cost:  12.048539 | test_cost:  16.515399 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6f9c3d4491b949f0a1835fe3302475f3","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"18656d9f4d0e48f58431fac20f154dbd","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 19/20 | train_cost:  11.554275 | test_cost:  16.070095 |  "]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"508e91fb69594ed18b2e7a5df85e966c","version_major":2,"version_minor":0},"text/plain":["Train:   0%|          | 0/153 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"562ad1e34c8c482394cd8c42322a3287","version_major":2,"version_minor":0},"text/plain":["Test:   0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Epoch: 20/20 | train_cost:  11.099799 | test_cost:  15.660364 |  "]}],"source":["epochs = 20\n","train_cost, test_cost = [], []\n","for i in range(epochs):\n","    cost = loop_fn(\"train\", train_set, trainloader, model, criterion, optimizer, device)\n","    train_cost.append(cost)\n","    with torch.no_grad():\n","        cost = loop_fn(\"test\", test_set, testloader, model, criterion, optimizer, device)\n","        test_cost.append(cost)\n","    \n","    print(f\"\\rEpoch: {i+1}/{epochs} | train_cost: {train_cost[-1]: 4f} | test_cost: {test_cost[-1]: 4f} | \", end=\" \")"]},{"cell_type":"markdown","metadata":{},"source":["# Cost History"]},{"cell_type":"code","execution_count":20,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T07:45:12.930658Z","iopub.status.busy":"2024-07-13T07:45:12.929977Z","iopub.status.idle":"2024-07-13T07:45:12.934826Z","shell.execute_reply":"2024-07-13T07:45:12.933785Z","shell.execute_reply.started":"2024-07-13T07:45:12.930625Z"},"trusted":true},"outputs":[],"source":["import matplotlib.pyplot as plt"]},{"cell_type":"code","execution_count":21,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T07:45:13.522396Z","iopub.status.busy":"2024-07-13T07:45:13.521790Z","iopub.status.idle":"2024-07-13T07:45:13.753671Z","shell.execute_reply":"2024-07-13T07:45:13.752801Z","shell.execute_reply.started":"2024-07-13T07:45:13.522364Z"},"trusted":true},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAjIAAAGwCAYAAACzXI8XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABca0lEQVR4nO3dd1zU9R8H8NexQZaggAgqmntvUXOBO7eZab80V5pampWjHGmFOzN3ObIcaTlyZbhwm4vcOMKVomkCAjKE7++PN8NTQMbdfe+O1/PxuEd33/ve997nSbz8fj+fz1ujKIoCIiIiIhNkoXYBRERERHnFIENEREQmi0GGiIiITBaDDBEREZksBhkiIiIyWQwyREREZLIYZIiIiMhkWaldgL6lpKTgzp07cHJygkajUbscIiIiygFFUfD48WN4e3vDwiLr8y5mH2Tu3LkDX19ftcsgIiKiPLh16xZ8fHyyfN7sg4yTkxMA+YNwdnZWuRoiIiLKiejoaPj6+qb/Hs+K2QeZtMtJzs7ODDJEREQm5mXDQjjYl4iIiEwWgwwRERGZLAYZIiIiMllmP0aGiIhIH5KTk5GUlKR2GSbL2toalpaW+T4OgwwREVEuKIqCiIgIREZGql2KyXN1dYWXl1e+1nkzmiAzdepUjB07Fh988AHmzJkDAIiPj8eoUaOwdu1aJCQkoHXr1liwYAE8PT3VLZaIiAqstBDj4eEBBwcHLraaB4qiIC4uDvfv3wcAFCtWLM/HMoogc/z4cSxevBjVqlXT2j5y5Ehs27YN69evh4uLC4YNG4auXbvi0KFDKlVKREQFWXJycnqIcXd3V7sck2Zvbw8AuH//Pjw8PPJ8mUn1wb4xMTHo3bs3vvvuOxQuXDh9e1RUFJYuXYrZs2ejRYsWqF27NpYvX47Dhw/j6NGjKlZMREQFVdqYGAcHB5UrMQ9pf475GWukepAZOnQo2rdvj8DAQK3tJ0+eRFJSktb2ChUqoESJEjhy5EiWx0tISEB0dLTWjYiISJd4OUk3dPHnqOqlpbVr1+LUqVM4fvz4C89FRETAxsYGrq6uWts9PT0RERGR5TGDgoLw+eef67pUIiIiMkKqnZG5desWPvjgA6xatQp2dnY6O+7YsWMRFRWVfrt165bOjk1ERETGRbUgc/LkSdy/fx+1atWClZUVrKysEBISgrlz58LKygqenp5ITEx8YXrbvXv34OXlleVxbW1t0/sqsb8SERGR/pQqVSp9prFaVAsyAQEBOHv2LEJDQ9NvderUQe/evdPvW1tbY/fu3emvCQsLw82bN+Hv769W2ekUBdi+HUhJUbsSIiKi7Gk0mmxvkyZNytNxjx8/jkGDBum22FxSbYyMk5MTqlSporWtUKFCcHd3T9/ev39/fPjhh3Bzc4OzszOGDx8Of39/NGjQQI2Stbz5JvDzz8D8+cB776ldDRERUdbu3r2bfv/nn3/GhAkTEBYWlr7N0dEx/b6iKEhOToaV1csjQtGiRXVbaB6oPmspO19//TVee+01dOvWDU2aNIGXlxc2bNigdlkAgMaN5b+jRwM3b6pbCxERqUhRgNhYdW6KkqMSvby80m8uLi7QaDTpjy9dugQnJyfs2LEDtWvXhq2tLQ4ePIhr166hU6dO8PT0hKOjI+rWrYtdu3ZpHff5S0sajQbff/89unTpAgcHB5QtWxa//fabLv+0X2AUC+Kl2bdvn9ZjOzs7zJ8/H/Pnz1enoGy89x6wdi1w6BAweDCwbRvA2XhERAVQXBzwzBkNg4qJAQoV0smhxowZg5kzZ6J06dIoXLgwbt26hXbt2uHLL7+Era0tVq5ciQ4dOiAsLAwlSpTI8jiff/45pk+fjhkzZuDbb79F7969cePGDbi5uemkzucZ9RkZY2ZhAXz/PWBjA+zYAaxerXZFREREeTd58mS0bNkSZcqUgZubG6pXr453330XVapUQdmyZTFlyhSUKVPmpWdY+vbtizfffBOvvPIKvvrqK8TExODPP//UW91GdUbG1FSoAEyYAHz2GfDBB0CrVoARXC4kIiJDcnCQMyNqvbeO1KlTR+txTEwMJk2ahG3btuHu3bt4+vQpnjx5gpsvGU/xbLuhQoUKwdnZOb2nkj4wyOTTJ58A69YBZ85ImOGZGSKiAkaj0dnlHTUVeu4zfPTRRwgODsbMmTPxyiuvwN7eHt27d0diYmK2x7G2ttZ6rNFokKLHKb68tJRP1tbA0qVyqWnNGmDrVrUrIiIiyr9Dhw6hb9++6NKlC6pWrQovLy9cv35d7bJewCCjA3XqAB9+KPcHDwbY3omIiExd2bJlsWHDBoSGhuKvv/5Cr1699HpmJa8YZHTk88+BMmWAf/6RKdlERESmbPbs2ShcuDAaNmyIDh06oHXr1qhVq5baZb1Aoyg5nIRuoqKjo+Hi4oKoqCi9tyvYuxdo0ULuh4QATZro9e2IiMjA4uPjER4eDj8/P532CSyosvvzzOnvb56R0aHmzYGBA+X+gAHAkyfq1kNERGTuGGR0bPp0oFgx4MoVYPJktashIiIybwwyOubqCixcKPdnzABOnVK1HCIiIrPGIKMHnToBr78OJCcD/fsDSUlqV0RERGSeGGTy6swZoHNn4MaNTJ/+9lugcGEgNBSYNcuglRERERUYDDJ5NWoUsHkzMHFipk97egJffy33J00CLl82XGlEREQFBYNMXn31lfx35Urg7NlMd3n7bem/lJAgs5iMcB0hIiIik8Ygk1d168pAGEUBxo3LdBeNBli8WFpwHDgALFli4BqJiIjMHINMfnz5JWBpKQ2WDhzIdJdSpTJO3nzyCXD7tuHKIyIiMncMMvlRtmzGCnijR8vZmUwMHQo0aAA8fiy9mMx7LWUiIjI2Go0m29ukSZPydexNmzbprNbcYpDJrwkTAAcH4MgRGfybCUtL6ZBtYwNs2wasXWvgGomIqEC7e/du+m3OnDlwdnbW2vbRRx+pXWKeMcjkV7FiGa2vx44Fnj7NdLdKlYBPP5X7778PPHhgoPqIiKjA8/LySr+5uLhAo9FobVu7di0qVqwIOzs7VKhQAQsWLEh/bWJiIoYNG4ZixYrBzs4OJUuWRFBQEACgVKlSAIAuXbpAo9GkPzYkK4O/ozn6+GNZzvfSJeCHH2QVvEyMGQOsXw+cOweMGAH89JNhyyQiIt1TFCAuTp33dnCQiSX5sWrVKkyYMAHz5s1DzZo1cfr0aQwcOBCFChVCnz59MHfuXPz2229Yt24dSpQogVu3buHWrVsAgOPHj8PDwwPLly9HmzZtYGlpqYNPlTsMMrrg7Ax89hkwcqSsK9OrF2Bv/8JuNjZyicnfH1i1SnZr106FeomISGfi4gBHR3XeOyZGZsbmx8SJEzFr1ix07doVAODn54cLFy5g8eLF6NOnD27evImyZcuicePG0Gg0KFmyZPprixYtCgBwdXWFl5dX/grJI15a0pUhQ4CSJYF//pFlfbNQr56cjQFk4O/jx4Ypj4iI6HmxsbG4du0a+vfvD0dHx/TbF198gWvXrgEA+vbti9DQUJQvXx7vv/8+/vjjD5Wr1sYzMrpiawtMmSKr4AUFyQp4bm6Z7jp5MrBxIxAeLsNq5s0zcK1ERKQzDg5yZkSt986PmNTCv/vuO9SvX1/rubTLRLVq1UJ4eDh27NiBXbt2oUePHggMDMQvv/ySvzfXEQYZXerVC5g5U/owTZ0KTJ+e6W6FCgHffQcEBgLz5wM9ewKNGxu4ViIi0gmNJv+Xd9Ti6ekJb29v/P333+jdu3eW+zk7O+ONN97AG2+8ge7du6NNmzb477//4ObmBmtrayQnJxuwam28tKRLlpZyNgYA5s4FUgdDZSYgAOjXT+4PGADExxugPiIioud8/vnnCAoKwty5c3H58mWcPXsWy5cvx+zZswEAs2fPxpo1a3Dp0iVcvnwZ69evh5eXF1xdXQHIzKXdu3cjIiICjx49Mnj9DDK61rYt0LSpNFh6yQJDM2cCXl5AWJhclSIiIjK0AQMG4Pvvv8fy5ctRtWpVNG3aFCtWrICfnx8AwMnJCdOnT0edOnVQt25dXL9+Hdu3b4eFhUSIWbNmITg4GL6+vqhZs6bB69coinmvMxsdHQ0XFxdERUXB2dnZMG967Jgs5WthIQ0lK1XKcteNG4GuXQErK+D4caBGDcOUSEREuRcfH4/w8HD4+fnBzs5O7XJMXnZ/njn9/c0zMvpQv76kk5SULBtKpunSBejWTdbR698/y/X0iIiIKBMMMvry1VcyZmbzZuDQoWx3nTcPcHUFTp0Cvv7aMOURERGZAwYZfSlfPmM0bzYNJQEZJ5M6pgoTJgBXrhigPiIiIjPAIKNPEyfKCr+HDgFbt2a7a9++Mh07Pl4aaqekGKZEIiIiU8Ygo0/Fi2cs4zt2LJDNPHuNBliyRBY3CgkBvv/eMCUSEVHumfk8GYPRxZ8jg4y+ffIJULgwcP488OOP2e7q5wd8+aXc//hj6XZARETGw9raGgAQp1aXSDOT9ueY9ueaF5x+bQizZgEffQT4+ACXL2faUDJNcjLQqJHM4O7YEdi0Kf+dTYmISHfu3r2LyMhIeHh4wMHBARr+TzrXFEVBXFwc7t+/D1dXVxQrVuyFfXL6+5tBxhDi44Fy5WSl3xkzJNRk49w5oFYtIClJLjH172+gOomI6KUURUFERAQiIyPVLsXkpXXNziwMMsikMoogAwArVgDvvCOXmf7+W+ZbZ2PqVBlWY28P/PknUKWKQaokIqIcSk5ORlJSktplmCxra+v0xpSZYZBJZTRBJjkZqF5dxsqMGZPRkykLKSlAu3bAzp1AxYqy6q+pNiUjIiLKLa7sa2yebSg5Z85LR/JaWAArVwLe3sDFi8DQofovkYiIyNQwyBjSa68BjRvLmJnPP3/p7h4ewJo1Emp++EGuThEREVEGBhlD0miAadPk/tKlwKVLL31JkyYZmWfoUODCBT3WR0REZGIYZAytYUOgU6ccNZRMM3asrPobFwf06CH/JSIiIgYZdXz1lVwv2rgROHr0pbtbWgI//SQ9mc6fB4YPN0CNREREJoBBRg2VKslUbEBW/s3BxDFPT2D1ask/y5a9dJFgIiKiAoFBRi2TJgF2dsCBA8D27Tl6SfPm0h0bAIYMydEQGyIiIrPGIKMWHx/g/ffl/ksaSj7rs8+AFi2A2FgZL/PkiR5rJCIiMnIMMmoaM0ZW+D17Fli1KkcvsbSUXT095WUffKDfEomIiIwZg4yaCheWszEAMH68rC+TA15eEmY0GuC772TsDBERUUHEIKO24cOB4sWBmzeBhQtz/LKAALnMBADvvitNtYmIiAoaVYPMwoULUa1aNTg7O8PZ2Rn+/v7YsWNH+vPNmjWDRqPRug0ePFjFivXA3j5jxbsvvgCionL80okTgaZNgZgYGS+TwxM6REREZkPVIOPj44OpU6fi5MmTOHHiBFq0aIFOnTrh/Pnz6fsMHDgQd+/eTb9Nnz5dxYr1pE8f6Qz533/AjBk5fpmlpVxWKloU+OsvYORIPdZIRERkhFQNMh06dEC7du1QtmxZlCtXDl9++SUcHR1x9JlF4hwcHODl5ZV+U7WDtb5YWWU0lJw9G7h7N8cv9faWxfI0GmDRIuDnn/VUIxERkREymjEyycnJWLt2LWJjY+Hv75++fdWqVShSpAiqVKmCsWPHIu4l6/MnJCQgOjpa62YSOnaU9gVPnuSooeSzWrXKGDM8cCBw9aoe6iMiIjJCGkXJwbKyenT27Fn4+/sjPj4ejo6OWL16Ndq1awcAWLJkCUqWLAlvb2+cOXMGo0ePRr169bBhw4Ysjzdp0iR8nkkQiIqKMv6zOQcOSJdIS0vpDlmuXI5f+vSprC9z4ABQsyZw+LCst0dERGSKoqOj4eLi8tLf36oHmcTERNy8eRNRUVH45Zdf8P333yMkJASVKlV6Yd89e/YgICAAV69eRZkyZTI9XkJCAhISEtIfR0dHw9fX1zSCDAB06ABs3Qp07w6sX5+rl96+DdSoATx8KJ2y583TT4lERET6ZjJB5nmBgYEoU6YMFi9e/MJzsbGxcHR0xO+//47WrVvn6Hg5/YMwGufOAdWqSf+lY8eAevVy9fIdO4DUE1pYv17yEBERkanJ6e9voxkjkyYlJUXrjMqzQkNDAQDFihUzYEUGVqWKzGICgNGjc9RQ8llt28rLAKB/f+DaNR3XR0REZERUPSMzduxYtG3bFiVKlMDjx4+xevVqTJs2DTt37kTp0qXTx8u4u7vjzJkzGDlyJHx8fBASEpLj9zC5MzKALI5XrhyQkAD88gvQrVuuXp6UBDRrJuNkatcGDh0CbG31UyoREZE+mMQZmfv37+Ptt99G+fLlERAQgOPHj2Pnzp1o2bIlbGxssGvXLrRq1QoVKlTAqFGj0K1bN2zZskXNkg2jRAng44/l/rBhQGRkrl5ubQ2sXQu4uQEnTwKffKL7EomIiIyB0Y2R0TWTPCMDyDK9NWoAYWEyp3rJklwfYutWGTsMAL/+CnTtqtsSiYiI9MUkzshQNuzspCMkIP/dty/Xh3jtNeCjj+R+v35AeLjuyiMiIjIGDDLG7NVXgbTeUoMGyWJ5ufTVV0CDBtLC6Y03gMREHddIRESkIgYZYzd1qvQhuHIFmDIl1y9PGy9TuDBw/HjGjCYiIiJzwCBj7FxcgAUL5P706dIdMpdKlgRWrJD7c+YAmzfrrDoiIiJVMciYgk6dZGW75GRgwADpR5BLHTtmdMfu2xe4cUO3JRIREamBQcZUfPst4OoKnDgBzJ2bp0NMnSoLBUdGcrwMERGZBwYZU+HlBcycKfc/+wz4++9cH8LGRsbLuLhI94O0jtlERESmikHGlPTrBzRvLrOXBg/OdfsCAPDzA5Yvl/uzZ2fcJyIiMkUMMqZEowEWL5Y1ZoKDgZUr83SYLl2AcePk/qBBwN69OqyRiIjIgBhkTE3ZssCkSXJ/5Ejg3r08HWbKFBkn8/SprPgbFqa7EomIiAyFQcYUjRoF1KwJPHoEjBiRp0NYWMhlpQYNZPBv+/bAgwc6rZKIiEjvGGRMkZUV8P33gKWljN7dujVPh7G3lzVl/PyAa9eAzp2lxRMREZGpYJAxVbVqAR9+KPeHDAEeP87TYTw8JAe5uACHDgH9++dpDDEREZEqGGRM2aRJQOnSwO3bGaN386BSJeCXX+REz+rVGUNwiIiIjB2DjClzcACWLJH78+cDhw/n+VCBgcDChXJ/8mTgp590UB8REZGeMciYuoAA4J135HrQgAFAQkKeDzVgAPDJJ3K/f3/gwAEd1UhERKQnDDLmYOZMGexy8aL0IciHoCCgWzdpX9C5szTdJiIiMlYMMubAzU16MQHAl18CFy7k+VAWFrLOXt26wH//ybTshw91VCcREZGOMciYi9dfBzp0AJKS5BpRSkqeD+XgAPz2G1CihJyR6do1X1esiIiI9IZBxlxoNMCCBYCTE3DkSMbI3Tzy8gK2bZPD7d8vrQw4LZuIiIwNg4w58fHJGCMzZgxw61a+DlelCrB+vay7t3KlXLUiIiIyJgwy5mbwYKBhQyAmRhbKy+dplNatgXnz5P748cCaNTqokYiISEcYZMyNhQXw3XeAjY1cG/r553wfcvDgjEWE33knX8vVEBER6RSDjDmqVAn49FO5//77Opl2NH060KmTDPrt1El6MxEREamNQcZcjRkDVK4M/PuvdMvOJ0tLYNUqafH04AHw2mvSfJuIiEhNDDLmysZGOmRrNMAPPwDBwfk+ZKFCwJYtMqb40iWge3dZOI+IiEgtDDLmrEEDYPhwuf/uu0BsbL4P6e0t3bIdHYE9e3QynpiIiCjPGGTM3RdfyMp24eHAxIk6OWT16jKG2MICWLYMmDZNJ4clIiLKNQYZc+fkBCxaJPe//ho4cUInh23XDvjmG7k/dqysN0NERGRoDDIFQdu2QK9e0rZgwABpY6ADw4bJpCgAePtt4NgxnRyWiIgoxxhkCoo5cwB3d+Cvv4BZs3R22NmzpbFkfDzQsSNw/brODk1ERPRSDDIFRdGicmkJACZNkm6QOmBpKav9Vq8O3L8voSYqSieHJiIieikGmYLkrbeAVq1kVbuBA/PVIftZTk4yk8nbG7hwQRpx6+jqFRERUbYYZAoSjUYG/jo4ACEhMuVIR3x8ZI0ZBwdZsmbYME7LJiIi/WOQKWj8/GRKNiANlHTYa6BWLbnMpNEAS5ZwWjYREekfg0xB9P77QOPGwOPHwBtvyKUmHenYUQYAAzIte/58nR2aiIjoBQwyBZGlJbB6NeDmBpw8KX2ZdGjEiIyelcOG6fQKFhERkRYGmYLK1xdYsULuz5kDbN6s08NPmQKMHCn3BwyQS05ERES6xiBTkHXoIONkAKBvX+DGDZ0dWqOR5WrefVcG/f7vf8DGjTo7PBEREQAGGQoKAurVAyIjgTff1Om8aY0GWLBAQkxysgzH+f13nR2eiIiIQabAs7EB1q4FXFyAI0eA8eN1evi0xpJpa8t06QLs3avTtyAiogKMQYZkSvbSpXJ/2jSdnzaxsgJ++kmuZMXHy38PH9bpWxARUQHFIEOiWzdg6FC5/7//AXfu6PTwNjbAunVAy5ZAbKz0sTx5UqdvQUREBRCDDGWYOROoUQN48EC6ZScn6/TwdnbApk3Aq68C0dHSLeHcOZ2+BRERFTAMMpTBzg74+WfA0VFaGEyZovO3cHCQvkz16gH//QcEBgJhYTp/GyIiKiAYZEhbuXLA4sVyf/JkYM8enb+Fs7MMw6lRA7h3DwgIAMLDdf42RERUADDI0It69QL695cFYHr3lrShY4ULA3/8AVSsCPzzD9CiBXD7ts7fhoiIzByDDGVu7lygUiUgIkIG/6ak6PwtihYFdu8GypQBrl+XMzN6yExERGTGVA0yCxcuRLVq1eDs7AxnZ2f4+/tjx44d6c/Hx8dj6NChcHd3h6OjI7p164Z7/E1nGA4OMs3I3h4IDtZbK+tixSTMlCgBXL4sY2YePtTLWxERkRlSNcj4+Phg6tSpOHnyJE6cOIEWLVqgU6dOOH/+PABg5MiR2LJlC9avX4+QkBDcuXMHXbt2VbPkgqVyZWDePLk/fjxw8KBe3qZkSQkzxYrJLKZWrWShYSIiopfRKIqiqF3Es9zc3DBjxgx0794dRYsWxerVq9G9e3cAwKVLl1CxYkUcOXIEDRo0yNHxoqOj4eLigqioKDg7O+uzdPOU1ihp1SrAxwcIDQXc3fXyVhcvAk2bAv/+C/j7yxgaR0e9vBURERm5nP7+NpoxMsnJyVi7di1iY2Ph7++PkydPIikpCYGBgen7VKhQASVKlMCRI0eyPE5CQgKio6O1bpQPGg2wcCFQtqyMxu3bV8KNHlSsKFexXF2lW0LHjsCTJ3p5KyIiMhOqB5mzZ8/C0dERtra2GDx4MDZu3IhKlSohIiICNjY2cHV11drf09MTERERWR4vKCgILi4u6TdfX189f4ICwMlJxsvY2soiMHPm6O2tqlcHdu6Ut9y7F+jaFUhI0NvbERGRiVM9yJQvXx6hoaE4duwYhgwZgj59+uDChQt5Pt7YsWMRFRWVfrt165YOqy3AatQAvv5a7o8eDfz5p97eql49YNs2GWf8++86b8pNRERmRPUgY2Njg1deeQW1a9dGUFAQqlevjm+++QZeXl5ITExE5HOjPu/duwcvL68sj2dra5s+CyrtRjoyeDDQvbukijfe0OuI3FdfBX77TU4CbdwI9Omj844JRERkBlQPMs9LSUlBQkICateuDWtra+zevTv9ubCwMNy8eRP+/v4qVliAaTTAd99Jt+zr14EBA/Q2XgaQqdi//CLds9esAQYN0styNkREZMJUDTJjx47F/v37cf36dZw9exZjx47Fvn370Lt3b7i4uKB///748MMPsXfvXpw8eRLvvPMO/P39czxjifTA1VX6MVlbA7/+KgOB9ei11yTEWFgAy5YBH3yg1+xEREQmRtUgc//+fbz99tsoX748AgICcPz4cezcuRMtW7YEAHz99dd47bXX0K1bNzRp0gReXl7YsGGDmiUTANStm7FA3siRMiVbj7p3B1askBNC8+bJEB2GGSIiAoxwHRld4zoyeqIoMj9661aZmn3ypEw10qMlS4B335X7kyYBEyfq9e2IiEhFJreODJkYjUZOk/j4AFeuyEBgPWfiQYMyJk5NmgRMmMAzM0REBR2DDOWdu7sMYLG0BFavBpYv1/tbjhgBTJ0q96dMAd57j7OZiIgKMgYZyp/GjSVRAMCwYUBqnyx9Gj0aWLBATgotWiTrzHDRPCKigolBhvJv9GigZUvpJ9CjBxAXp/e3HDIEWLtWJk+tXy+zmx4/1vvbEhGRkWGQofyzsAB+/BHw8gIuXACGDzfI2/boAWzfDhQqBOzaBbRoIQ0niYio4GCQId3w9JQO2RqNLPjy008GedvAQOnJ5O4OnDghKwLfuGGQtyYiIiPAIEO606IFMH683B88GAgLM8jb1q0LHDwI+PrKWzZqZJChOkREZAQYZEi3JkwAmjYFYmOBLl302o/pWRUqAIcPAxUrAv/8I2dmjh41yFsTEZGKGGRIt9KmYhcvDly8mNFk0gB8fIADB4D69YFHj4CAAOmeTURE5otBhnTP2xvYskVG4e7eLYu9GGjlOnd3ecvWrWXyVIcOkquIiMg8MciQftSsmdHt8fvvgZkzDfbWhQoBv/0m68s8fQr07g18+63B3p6IiAyIQYb0p0MHYPZsuT96NGDAhp82NjJxatgwefz++2xpQERkjhhkSL/efz/j0tJbbwHHjxvsrS0sgLlzgcmT5fGUKbKQHlsaEBGZDwYZ0i+NBvjmG6BNG1n5t2NH4OZNg779+PHAwoVyf/FioGdPtjQgIjIXDDKkf1ZWwM8/A1WrAhER0k8gOtqgJQweLCVYWwO//AK0b8+WBkRE5oBBhgzD2RnYulVWAD57Vk6LPH1q0BJefz2jpcHu3UDz5mxpQERk6hhkyHBKlJBp2fb2wI4dwMiRBi8hraVBkSLAyZPSvJstDYiITBeDDBlW3boZfZjmzZPRuCqUcPCg5KrLl4GGDdnSgIjIVDHIkOF17QpMmyb3R46US04GVr48cOgQUKkScOeOtDQ4csTgZRARUT4xyJA6Pv4YGDAASEmR8TKhoQYvwccH2L8faNBAWhoEBsoVLyIiMh0MMqQOjQZYsEAaIsXGykymO3cMXoa7O7Brl8wOj4uT2eGrVhm8DCIiyiMGGVJP2lzoChWkZXWHDhJqDKxQIWDz5oyWBm+9JUvfEBGR8WOQIXW5ugLbtgFFiwKnTkljJBWW3k1raTB8uDweMULWnklMNHgpRESUCwwypL7SpYFNmwBbWzk1Mnq0KmVYWMiZmKlTM1YBDggA7t1TpRwiIsoBBhkyDg0bAitWyP1ZsyRFqECjkRy1ZYus4XfwIFC7NnDihCrlEBHRSzDIkPHo2VM6OwLA0KHAH3+oVkr79sCff8o07X/+kYXzfvxRtXKIiCgLDDJkXD79FHj7bRkn8/rrwLlzqpVSvjxw7JhMqEpIkLI+/NDgnRWIiCgbDDJkXDQaYMkSoEkTaSz52muqDlJxcZFhO599Jo+//lqmaj98qFpJRET0DAYZMj62tsCGDUDZstIIqVMn4MkT1cqxsJArXr/8ktFwsm5d4MwZ1UoiIqJUDDJknNzdZVq2m5tc3+nTR1YBVlG3btLGoHRpIDwc8PeXcENEROphkCHjVbYssHGjLJy3fj0wfrzaFaFqVeD4cWlnEBcnw3g+/VT1jEVEVGAxyJBxa9IE+P57uf/VV8Dy5erWAzlJtGMHMGqUPP7qK2ltEBWlbl1ERAURgwwZv7ffzhhtO2gQsHevuvUAsLICZs6UKdl2dnIVrH594NIltSsjIipYGGTINEyeLOvMPH0qg1XCwtSuCID0ZTp4EPD1lZLq1we2blW7KiKigoNBhkyDRiOXlfz9gUePZMW6f/9VuyoAGSv/vvqqzBjv2BH44gtAUdSujIjI/DHIkOmws5NFXfz8gGvXgJYtgf/+U7sqAICHB7BrF/DeexJgxo+XgcAxMWpXRkRk3vIUZFauXImEhIQXticmJmLlypX5LoooS0WLAr//Dnh6An/9BbRqBURGql0VAOmgPX8+8N13MtHq11+lhdTff6tdGRGR+dIoSu5PgFtaWuLu3bvw8PDQ2v7w4UN4eHggOTlZZwXmV3R0NFxcXBAVFQVnZ2e1yyFduXABaNZMLi/Vry99mYzo+z18WIbyREQAhQsD69bJlG0iIsqZnP7+ztMZGUVRoNFoXth++/ZtuLi45OWQRLlTqZJcy0lbMK9dO6O6jtOwoYybqVdPhvS0bg3Mns1xM0REumaVm51r1qwJjUYDjUaDgIAAWFllvDw5ORnh4eFo06aNzoskylS1akBwMBAQABw6BHToIPOgHRzUrgwAULw4EBICDBkCrFgh686cPi2tpOzt1a6OiMg85CrIdO7cGQAQGhqK1q1bw9HRMf05GxsblCpVCt26ddNpgUTZqlVLLisFBgL79klfpi1bZGCwEbCzA5YtkzJHjgR++gm4eFEWLPb1Vbs6IiLTl6cxMj/88AN69uwJW1tbfdSkUxwjU0AcPizXb2JigLZtJSkY2d/PvXtlJtPDh3JFbMkSGUdDREQv0usYmRYtWuDfZ9bw+PPPPzFixAgsWbIkL4cjyr+GDTMuK+3YIYkhMVHtqrQ0by7jZmrXllnj3bsD77wja88QEVHe5CnI9OrVC3tTl4mPiIhAYGAg/vzzT3z66aeYPHmyTgskyrEmTTIuK23ZArz5JpCUpHZVWkqVkpNH48bJGn8rVgA1asgQHyIiyr08BZlz586hXr16AIB169ahatWqOHz4MFatWoUVK1bosj6i3GnRAti0SRZ12bBB+jQ9fap2VVpsbIAvv5SBwCVLAuHhksHGjze63EVEZPTyFGSSkpLSx8fs2rULHTt2BABUqFABd+/e1V11RHnRurWsRmdtDaxdC/TrBxjR2kZpXn1V1vR7+20gJUXaGjRsCFy+rHZlRESmI09BpnLlyli0aBEOHDiA4ODg9CnXd+7cgbu7u04LJMqT116TVeisrKRF9aBBkhaMjIsL8MMPwM8/y8J5J04ANWsCixZxzRkiopzIU5CZNm0aFi9ejGbNmuHNN99E9erVAQC//fZb+iUnItV17gysXg1YWMgc6LRGSEaoRw/g7FmZRR4XJ2vPdOgA3LundmVERMYtT0GmWbNmePDgAR48eIBly5albx80aBAWLVqU4+MEBQWhbt26cHJygoeHBzp37oywsLAX3ittEb602+DBg/NSNhVEr78uZ2Q0GmDxYuCDD4w2zBQvDuzcCXz9tcwc37YNqFpVxi0TEVHm8tz92tLSEk+fPsXBgwdx8OBB/PvvvyhVqtQL/ZeyExISgqFDh+Lo0aMIDg5GUlISWrVqhdjYWK39Bg4ciLt376bfpk+fnteyqSDq1QtYvlzCzLffAh9/bLRhxsICGDECOH5cQsy//wIdOwKDBwPP/VgQERHyGGRiY2PRr18/FCtWDE2aNEGTJk3g7e2N/v37Iy4uLsfH+f3339G3b19UrlwZ1atXx4oVK3Dz5k2cPHlSaz8HBwd4eXml37iwHeVanz5yRgYAZs0CPv3UaMMMICHm+HHgo4/k8eLFMnbm+HF16yIiMjZ5CjIffvghQkJCsGXLFkRGRiIyMhKbN29GSEgIRo0aledioqKiAABubm5a21etWoUiRYqgSpUqGDt2bLZhKSEhAdHR0Vo3IgDAwIHA/PlyPygIMPI1j2xtgRkzgN275bLTlSuAvz8wZYrRzSgnIlJNnloUFClSBL/88guaNWumtX3v3r3o0aOH1qq/OZWSkoKOHTsiMjISBw8eTN++ZMkSlCxZEt7e3jhz5gxGjx6NevXqYcOGDZkeZ9KkSfj8889f2M4WBZRuzhxpfATIgi7jxqlaTk48eiQDgH/+WR77+8vQnzJl1K2LiEhfctqiIE9BxsHBASdPnkTFihW1tp8/fx716tV7YYxLTgwZMgQ7duzAwYMH4ePjk+V+e/bsQUBAAK5evYoymfxfPCEhAQkJCemPo6Oj4evryyBD2qZPB0aPlvszZmRcwzFiiiKTsN57T9oaODoCc+cCffvK8B8iInOi115L/v7+mDhxIuLj49O3PXnyBJ9//jn8/f1zfbxhw4Zh69at2Lt3b7YhBgDq168PALh69Wqmz9va2sLZ2VnrRvSCTz6RazSADP6dO1fdenJAowF69wbOnJGVgGNiZK2/7t2BBw/Uro6ISB15CjJz5szBoUOH4OPjg4CAAAQEBMDX1xeHDh3CN998k+PjKIqCYcOGYePGjdizZw/8/Pxe+prQ0FAAQLFixfJSOlGGzz6TvgCATMteuFDdenKoZElgzx5g6lRZvHjDBqBaNZm6TURU0OTp0hIAxMXFYdWqVbh06RIAoGLFiujduzfs7e1zfIz33nsPq1evxubNm1G+fPn07S4uLrC3t8e1a9ewevVqtGvXDu7u7jhz5gxGjhwJHx8fhISE5Og9cnpqigooRQHGjgWmTZPH338P9O+vbk25cPq0nKW5eFEeDx8uHyUXP4ZEREZJr2NkgoKC4OnpiX79+mltX7ZsGf7991+MTht78BKaLC7sL1++HH379sWtW7fw1ltv4dy5c4iNjYWvry+6dOmCzz77LMehhEGGXkpRgFGjZCW6tJbUb7+tdlU59uSJDPf59lt5XLGitD2oW1fduoiI8kOvQaZUqVJYvXo1GjZsqLX92LFj6NmzJ8LDw3NfsZ4wyFCOKIqczpg/X1al++kn4M031a4qV37/HXjnHSAiQvLY0KHSiNLFRe3KiIhyT6+DfSMiIjIdo1K0aFF2vybTpNHIgN+05pJvvSWXmUxImzbSr+mttySXzZsnZ2fWrzfqtf+IiPIlT0EmbWDv8w4dOgRvb+98F0WkCgsLGfA7YICEmYEDZWaTCaWAIkVkfZngYKBsWeDuXWlI2b498PffaldHRKR7eQoyAwcOxIgRI7B8+XLcuHEDN27cwLJlyzBy5EgMHDhQ1zUSGY6FBbBkibQwAIAJE2ThluRkdevKpcBAmaY9cSJgYwPs2AFUriwLGicmql0dEZHu5GmMjKIoGDNmDObOnYvE1P8r2tnZYfTo0ZgwYYLOi8wPjpGhPJs/X8bNKArQpYusRmdnp3ZVuRYWJllszx55XKkSsGgR8Oqr6tZFRJQdvQ72TRMTE4OLFy/C3t4eZcuWha2tbV4PpTcMMpQvv/wi85sTE+U3/+bNQOHCaleVa4oi45dHjZKO2oAspjd9OuDurm5tRESZMUiQMQUMMpRvISFAx47SF6BKFblO85IVqI3Vf/8BY8YA330nj4sUAWbOlNnmbHNARMZEr7OWiAqUpk2BAwcAb2/g3DmgYUPgwgW1q8oTNzcZAnTwoGSyBw+kV1Pz5kDq2pZERCaFQYYoJ6pVAw4fBsqXB27dAho3lscmqlEj4NSpjFWAQ0LkI44fLwvsERGZCgYZopwqWRI4dAho0AB49AgICAB++03tqvLM2lp6Z164INOzk5JkAb2qVYE//lC7OiKinGGQIcoNd3dg927gtdeA+HiZzWRiC+c9r1QpYMsWGdfs7Q1cuwa0bi0LG0dEqF0dEVH2GGSIcsvBAdi4Uab9mOjCec/TaIBu3aT55AcfyHI6a9cCFSrIGoEpKWpXSESUOQYZorywspIzMSa+cN7znJ2BOXOAP/8EatcGoqLkYzVsCISGql0dEdGLGGSI8kqjkUEl8+bJ/UWLgNdfl0tOJq52beDYMWk/5eQk9+vUkXVoYmLUro6IKAODDFF+DR0KrFsnvQA2bgRatZLBwCbO0lIWNr54UfJZcjIwe7Y0oly1ipebiMg4MMgQ6UL37jLVx9lZ1pxp0gS4fVvtqnSieHHJadu2ycDg27elw3bdusDevWpXR0QFHYMMka6Y0cJ5mWnXDjh/HvjyS7ncdOoU0KKFTOA6f17t6oiooGKQIdIlM1s473kODsC4ccDVq3JFzcpKztRUqwYMGgTcvat2hURU0DDIEOmamS2clxkPDxnjfP68LKWTkiL9m8qWBSZN4oBgIjIcBhkifTDDhfMyU64csGGDXFGrXx+IjQU+/1wCzZIlwNOnaldIROaOQYZIX8xw4bysNG4MHDkig4JLl5YVgd99Vy45bd1qlh+ZiIwEgwyRPqUtnPfZZ/LYTBbOy4xGI9O0L16URfXc3OR+hw4yKPjkSbUrJCJzxCBDpG8ajZyJeXbhvC5dgOhotSvTCxsbaXNw7Zo0pbS1BfbtkwX1evcGrl9Xu0IiMicMMkSGkrZwnq2tdGmsV09OWZgpV1dg2jQgLEzWnQGA1atlQtfHH5vFmoFEZAQYZIgMqXt3YP9+wMdHfsPXqyejZc1YyZLAjz/KpaUWLYDERGDmTKBMGeDrr4GEBLUrJCJTxiBDZGj16slv9WbNZJ5yt27A2LFmOW7mWbVqAbt2ybozlSvLGZkPP5SWBz//zAHBRJQ3DDJEavDwAIKD5Tc5AEydCrRtCzx8qG5deqbRyArBoaGy7oyXFxAeDvTsKcvuHDigdoVEZGoYZIjUYmUFzJoFrFkjU7WDg2VE7OnTalemd1ZWwIABwJUrsoBeoULAn39Ki6qOHTnDiYhyjkGGSG09e8oiLKVLy5Sehg1lUEkB4OgITJwoLQ/efRewsJBx0HXqyJmbI0fUrpCIjB2DDJExqFYNOHFCLi/FxwNvvw28/z6QlKR2ZQbh5SWz0s+flxlOFhbAjh2S6QIDgZAQtSskImPFIENkLAoXlmVwJ0yQx99+K9N8IiLUrcuAKlSQk1FhYbIgspWVdHpo1kyai+/axUHBRKSNQYbImFhYSLOizZsBZ2fg4EGZ7lPArrG88gqwdKmMoRk8WBbZ278faNlSztJs385AQ0SCQYbIGHXsKKNfK1UC7t6V0xGLFhW4396lSgELF8oqwcOHA3Z2wNGjQPv2QN26wKZN0saKiAouBhkiY1W+vPzW7t5dxsoMGQL07y9jaAoYHx9g7lyZqj1qlEzyOnlSOj3UqCELJpv5MjxElAUGGSJj5uQkv6WnTZPLTsuXA6++Cty8qXZlqvDyklWBr1+XNQSdnICzZ4E33gCqVAF++gl4+lTtKonIkBhkiIydRiPdF3fuBNzdZXZT7drAnj1qV6aaokWBr76SQDNxovR1unQJ+N//ZKXgZcsKzIQvogKPQYbIVAQGSoipWRN48EBGvs6cWeDGzTzLzU0W1Lt+HfjyS8l5V6/KFbiyZWVYEXs5EZk3BhkiU1KqFHDokKwzk5IibaR79pSeTQWYiwswbpwEmhkzAE9P4MYNGVZUpozMZH/yRO0qiUgfGGSITI29PbBiBTB/viy0sm4d4O8vc5ULOEdH4KOPZFDwN98AxYsD//wjawv6+UlHiAKe+YjMDoMMkSnSaID33gP27ZMRsOfOyXzkrVvVrswo2NtLeLl2TaZvlywJ3LsnIcfXV05k3bihdpVEpAsMMkSmrFEjmYfcsCEQFQV06CAL6nFxFQCAra0sqHfliiyw98orQGSkDC0qXVpmth84UKCHGRGZPAYZIlPn7Q3s3StnaAAZ/dq6tVxTIQCAtbW0PAgLk6aUgYGS9X79VTpu164NrFzJgcFEpohBhsgc2NjImJnly+W6yq5dQNWqwM8/q12ZUbGwAF57DQgOlvVnBg6U1YJPnwb69JFLUJMmFaj2VkQmj0GGyJz07Su/levUAR49khlNvXvL9RTSUqUKsGQJcPs2EBQkA4Pv3ZMrcyVLSrA5dUrtKonoZRhkiMxN+fLA4cPA+PFyCmL1ajk7U4AX0MuOuzswZozMdFq7ViaAJSbKpabatWUh5V9/5YrBRMaKQYbIHFlbA5Mny5ozZcrIaYeAAGlUVAB7NeWEtbW0Ojh8GDh2DOjVS2a3Hzwog4LLlJE1ah49UrtSInoWgwyROWvQAAgNBQYNksezZ8s07b/+UrUsY1evHrBqlUzR/uwzoEgRaW/1ySfSwPK996QlAhGpj0GGyNw5OgKLFwO//QZ4eMiaM/XqyekFtozOlrc3MGUKcOuWTN+uVg2Ii5O1aSpWBNq0AXbs4Gx3IjUxyBAVFB06yFSdjh1lEMgnnwAtWnBluByws5Pp26GhMtO9c2dZk3DnTqBdO6BSJWDBAq4aTKQGVYNMUFAQ6tatCycnJ3h4eKBz584ICwvT2ic+Ph5Dhw6Fu7s7HB0d0a1bN9y7d0+liolMnIcHsGkT8P33QKFCwP79cpph5UquCpcDGg3QrBmwcaM0pxw5EnB2lvVphg6Vy07vv88rd0SGpGqQCQkJwdChQ3H06FEEBwcjKSkJrVq1QmxsbPo+I0eOxJYtW7B+/XqEhITgzp076Nq1q4pVE5k4jUbaQ//1l6wIHB0tc4179AAePlS7OpNRurQMObp9G5g7V1YNjoqSBpU1ashQpMWL5Y+XiPRHoyjG88+wf//9Fx4eHggJCUGTJk0QFRWFokWLYvXq1ejevTsA4NKlS6hYsSKOHDmCBg0avPSY0dHRcHFxQVRUFJydnfX9EYhMy9OnwLRpsgrc06dAsWKyqF7r1mpXZnJSUmShvaVL5aRXUpJsd3CQjDhggORGjUbVMolMRk5/fxvVGJmoqCgAgJubGwDg5MmTSEpKQmBgYPo+FSpUQIkSJXDkyJFMj5GQkIDo6GitGxFlwcoK+PRT4OhRoEIF4O5dGcE6fLiMaqUcs7CQ/LdunXSHmDVLBgTHxUmz8saN5fHMmcD9+2pXS2Q+jCbIpKSkYMSIEWjUqBGqVKkCAIiIiICNjQ1cXV219vX09EREFmuIBwUFwcXFJf3m6+ur79KJTF/t2tJ8cvhweTxvXsY2yrWiRYEPPwTOn5d1afr1kzMzYWHSebt4cVmb5vffOXGMKL+MJsgMHToU586dw9q1a/N1nLFjxyIqKir9duvWLR1VSGTmHBxksMfvv8slpkuXZB2aL77gsrZ5pNHISsFLl8rJriVLZOb706eyWnDbtoCfHzBxInD9utrVEpkmowgyw4YNw9atW7F37174+Pikb/fy8kJiYiIin+sTc+/ePXh5eWV6LFtbWzg7O2vdiCgXWreWadqvvy6/ccePlxbR166pXZlJc3aWJpXHjsk46/ffB9zcZI2ayZNl8HDapSl24SbKOVWDjKIoGDZsGDZu3Ig9e/bAz89P6/natWvD2toau3fvTt8WFhaGmzdvwt/f39DlEhUc7u7SOfvHH+U38JEjQPXqMm3beOYHmKxq1YBvvpGxNGvWSPcIRQH++EPaJBQvnnFpioiyp+qspffeew+rV6/G5s2bUb58+fTtLi4usLe3BwAMGTIE27dvx4oVK+Ds7IzhqdfwDx8+nKP34Kwlony6eVOmZ+/bJ487dpSlbb29VS3L3Pz9N7BsmUwau3MnY7u/v8x46tFDFmkmKihy+vtb1SCjyWIe4vLly9G3b18AsiDeqFGjsGbNGiQkJKB169ZYsGBBlpeWnscgQ6QDKSmyaMqnn8qqwE5OMnZm6FDA0lLt6szK06cyTGnpUmDLlozBwI6OcrbmrbfkSp+FUQwMINIfkwgyhsAgQ6RDZ89KA8qjR+VxrVrAokWy+hvpXEQE8MMPckXv6tWM7cWLS6jp1Uu+Aq5NQ+aIQSYVgwyRjqWkAN99B4wZA0RGym/R994DvvwScHFRuzqzpCjSTeLHH4FffpEVhNOUKyeB5s035T6RuWCQScUgQ6Qn9+7Joig//iiPvbyAr7+WUwU8RaA3CQnScXv1arn0FB+f8Vzt2hJq0gYME5kyBplUDDJEerZnj5yRSWv42rIlMH8+ULasunUVAI8fSzuE1aulPULaeJq05pa9egHdugGFC6tZJVHeMMikYpAhMoCEBGDGDBkAnJAA2NoCY8cCo0cDdnZqV1cg/PsvsH69hJpDhzK2W1vLwnu9egEdOsi6h0SmgEEmFYMMkQFdvSozmf74Qx6XLQssWAA80y+N9O/6dWDtWlmj5syZjO2FCgFdush4mpYtJeQQGSsGmVQMMkQGpihyamDECFmXH5DTAbNmyTgaMqhz5yTQrF6t3QbB3V3WpunVS7pyczo3GRsGmVQMMkQqiYqS9gbz58tMJxcXIChIpm9z7RmDUxRpj7B6tSza/GwHbl9fOUvTsydQowbHapNxYJBJxSBDpLKTJ4HBg4ETJ+RxvXqy9kzNmurWVYA9fQrs3Suh5tdfZdBwmlKlgM6dga5d5UwNMyephUEmFYMMkRFITpbwMm4cEB0t1zGGD5duify5VNWTJ8D27XL5aft2eZymaFGgUycZVxMQIGO4iQyFQSYVgwyREbl7V7ohrl0rj729pXtit268nmEE4uJknPaGDbJGTWRkxnNOTkC7dhJq2rWTx0T6xCCTikGGyAj98YesPXPtmjxu2xaYNw8oXVrduihdUhIQEgJs3Chr1TzbyNLWViaidekiPUSLFlWtTDJjDDKpGGSIjFR8PDB1qgwATkyU9WY++wz46CNewzAyKSnA8eNypmbjRuDKlYznLCyAxo0l1HTpApQsqV6dZF4YZFIxyBAZucuX5ezM7t3yuFw5YNo0GZzBy01GR1GACxck0GzcCJw6pf18rVoZoaZSJX6FlHcMMqkYZIhMgKLIaNORIzPmBTduLKsFN2igbm2UrevX5dLTxo3AwYNy9iZN2bISaLp2lQbpXKuGcoNBJhWDDJEJiY4Gpk8HZs/OmD7TvTvw1Vfs3WQC7t8HfvtNQs2uXXLFMI23N/DaazJQOCAAcHRUr04yDQwyqRhkiEzQP/8AEyYAK1bIP/GtrGQtmvHjAQ8PtaujHIiOli7dGzcC27YBMTEZz9nYAE2ayBjvdu2A8uV5CYpexCCTikGGyISdPQuMGSMLnAAy53f0aLkExe6HJiM+Xpqk79ghoSY8XPt5Pz8JNO3aSddufrUEMMikY5AhMgN79gCffCKrBANynWLyZKBvXy49a2IURcZ3b98uwSYkRPsSlJ2dhJm0YFOmjGqlksoYZFIxyBCZiZQUaRI0blxG98PKlWVMTdu2vDZhomJiJKdu3y63W7e0ny9XLuMSVJMmEnSoYGCQScUgQ2RmEhKABQuAKVOAR49kW/PmEmjq1FG3NsqXtKndaaHm4EHpC5XGwUEGCqcFG65ZY94YZFIxyBCZqUePZEG9b76RcANIC+cvv5RBF2TyoqNl9lPaZahnVxcGZJ2atEtQjRrJIGIyHwwyqRhkiMzczZsym+nHH+Wf9NbWwLBhwKefAu7ualdHOqIowJkzGWdrjhyRXqRpHB1lbE1AgNyqVOHVRlPHIJOKQYaogAgNlQHBwcHy2MVFxtMMHw7Y26taGuneo0fyVaedrUlbRzGNpyfQooWEmsBAXoYyRQwyqRhkiAqYP/6QQPPXX/LY1xf44gugd2/OcDJTKSmSY3fvltuBA9LJ+1llymSEmubNgSJFVCmVcoFBJhWDDFEBlJwMrFolTSjTpsFUry4Dglu1Urc20ruEBODo0Yxgc+yY9mUoAKhRQ0JNQADw6qtAoUKqlErZYJBJxSBDVIA9eQJ8+620OIiKkm3Nm8uYmmbNOIiigIiOBvbvl1Czaxdw7pz289bW0tIrLdjUqyfbSF0MMqkYZIgIDx/KbKZ584CkJNnWqJEEmlatGGgKmHv3ZO2aXbsk3Ny4of28oyPQtGnGwOGqVflXRA0MMqkYZIgo3c2bcnnp++8zpmzXrSuB5rXX+NuqAFIU4O+/M87W7NkjufdZHh5yIq9JE7lVqsRO3obAIJOKQYaIXnDnDjBjBrB4cUaX7Ro1ZExNly78LVWApaTINO+0szX79784cNjNTcbVvPqqBJuaNaWvKekWg0wqBhkiytL9+8CsWcD8+UBsrGyrXFnWoOnRg7OcCImJMnA4JERCzeHDLwabQoWAhg0zztjUq8dWCrrAIJOKQYaIXurhQ2DOHGDuXBkZCkiTn3HjgF69OPKT0iUlAadPS6jZv1/aKKR1ykhjYyNhpkkTOWvTsCHAXz+5xyCTikGGiHIsMlJmOc2ZA/z3n2zz8wPGjgX69OEa+PSClBTg/HkJNQcOyH/v3tXex8JCLj+lXYpq3BgoWlSdek0Jg0wqBhkiyrXHj6Ux5axZwL//yjZfX2D0aKB/f143oCwpCnDtWkao2b9fBhM/r1KljGDz6qvy14u0McikYpAhojyLjQWWLJGBwWn/zC5WDPj4Y+Ddd6UdM9FL/POPdrA5f/7FfUqUAPz95daggZzBKegnABlkUjHIEFG+xccDS5cC06ZlrBRctCgwahTw3nuAk5O69ZFJefhQxtakXY46derFlYdtbYFatbTDjY+POvWqhUEmFYMMEelMYiLwww9AUBAQHi7b3NyAESOkOaWrq5rVkYmKiQGOH5eO3keOyCypBw9e3M/HJyPU+PtL0LG1NXy9hsIgk4pBhoh0LikJWL1aWh9cvizbnJ0lzLz/vqygRpRHaeNsng02Z868eNbGxkYuQT171sbX13zWdWSQScUgQ0R6k5wMrFsn3bUvXJBttrbSafuDD4Bq1dStj8xGbKyctTl6NCPgpI1Df5a3d8YZm7SzNvb2hq9XFxhkUjHIEJHepaQAGzcCU6cCJ05kbG/eXC47tW/PxfVIpxRFrm4+e9YmNPTFszbW1rJodb16QJ06cqtY0TT+OjLIpGKQISKDURT5rTJnDrBhQ8ZvlTJl5JLTO+9wYDDpTVyc5Ohnz9rcu/fifg4OcqYmLdjUqQOULWt8nTkYZFIxyBCRKm7elNYHS5bIQnuAjKPp10/G0pQurWp5ZP4URTp7Hz0qAefECeDkSRlc/DwnJ6B2be1wU7q0uuNtGGRSMcgQkapiY4GVK4FvvgHCwmSbRgN06iSXnZo0MZ/RmWT0kpNlfHpasDlxQloupPVOfVbhwtrBpk4dww4mZpBJxSBDREYhJQXYuVMCzc6dGdtr1JBA07Onec+lJaP19Clw8aJ2uAkNldUGnle0qASaunUzwk2xYvqpi0EmFYMMERmdCxekQeXKlRn/FPbwkMX1Bg8GPD3VrY8KvMRE4Nw57XBz9qyEnud5ewOTJ0v3Dl1ikEnFIENERuvhQ+C774B582Qde0AWB3nzTTlLU6OGmtURaYmPl/Vsng0358/LycbVq+WvrS4xyKRikCEio5eUBPz6q8x2OnYsY3vTphJoOnQwjfmyVODExsplqAoVAHd33R6bQSYVgwwRmZSjR2Uczfr1GdO3/fxk+na/fjLziagAyOnvbyObNU5EVMA1aACsWSOrnY0ZI1NHwsOBkSOB4sWBQYOAP/+UubVEpG6Q2b9/Pzp06ABvb29oNBps2rRJ6/m+fftCo9Fo3dq0aaNOsUREhuTrK80pb98GFi2S5VhjYmRMTf36Mn5m3jzg0SO1KyVSlapBJjY2FtWrV8f8+fOz3KdNmza4e/du+m3NmjUGrJCISGUODsC778qoyn37pI+Tra2Muhw+XKaM/O9/QEgIz9JQgWSl5pu3bdsWbdu2zXYfW1tbeHl5GagiIiIjpdHI4N+mTYFvvwV++knOzpw9K/d/+gkoVw4YMADo04cduKnAMPoxMvv27YOHhwfKly+PIUOG4OHDh9nun5CQgOjoaK0bEZFZKVxYzsb89ZfMchowAChUSJZs/eQTGUvTvbssvPd8F0EiM2PUQaZNmzZYuXIldu/ejWnTpiEkJARt27ZFcjY/mEFBQXBxcUm/+fr6GrBiIiID0mikrfF33wF372aMn3n6VKZzt2kjDSsnT5axNkRmyGimX2s0GmzcuBGdO3fOcp+///4bZcqUwa5duxAQEJDpPgkJCUhISEh/HB0dDV9fX06/JqKC48wZ4PvvgR9/zGhYaWEhwWbgQKB9e8DaWtUSiV7GLKdfly5dGkWKFMHVq1ez3MfW1hbOzs5aNyKiAqVaNWmBcOeOjJ1p1kyWX92+HejSBShRAhg7Frh2Te1KifLNpILM7du38fDhQxTTV4cqIiJzYm8vs5z27s0YP+PhAUREAFOnAq+8AgQEyLo18fFqV0uUJ6oGmZiYGISGhiI0NBQAEB4ejtDQUNy8eRMxMTH4+OOPcfToUVy/fh27d+9Gp06d8Morr6B169Zqlk1EZHrKlgWmTZOxMr/+CrRtK2Ns9uwBevWSAcIffMDF9sjkqDpGZt++fWjevPkL2/v06YOFCxeic+fOOH36NCIjI+Ht7Y1WrVphypQp8MxFZ1i2KCAiysLNm8CyZXK7dStj+yuvyJmcXr1kSjeRCthrKRWDDBHRSyQnA8HBMjh40yYgLi7juTp1JNT07AlwTS8yIAaZVAwyRES5EBMD/PYbsGqV9jo0FhYynqZ3bxkwzP+fkp4xyKRikCEiyqN//wXWrZNQc+RIxnY7O6BDBwk1bdsCNjbq1Uhmi0EmFYMMEZEO/P03sHq1hJpLlzK2Fy4MvP66hJrGjeXMDZEOMMikYpAhItIhRQFOn5ZAs2aNrCicpkQJ4M03JdRUrapejWQWGGRSMcgQEelJcrJ05F61SqZ0P9vbrmpVmfXUq5cEHKJcYpBJxSBDRGQA8fHAtm0SarZtAxITM5579VU5S9OtG1CkiHo1kklhkEnFIENEZGCPHskZmlWrgJCQjAX2LC2Bpk2Brl2Bzp1lET6iLDDIpGKQISJS0e3bMpZmzRoZW/Msf38JNV27AqVLq1MfGS0GmVQMMkRERuLvv4GNG4ENG4DDh7Wfq1FDLj117QpUqqRKeWRcGGRSMcgQERmhO3dkFeFff5XLT2kL7wFAhQoZZ2pq1ZKeUFTgMMikYpAhIjJyDx7IasIbNkirhGcHCpcsKYGmWze5FMV1agoMBplUDDJERCYkKgrYvl3O1OzYod33yctL2iN07SqDhq2t1auT9I5BJhWDDBGRiYqLk35PGzYAW7ZIyEnj5gZ07CihpmVLaZtAZoVBJhWDDBGRGUhMBPbskVCzaZP0gUrj6Ai0by/9n9q0AdzdVSuTdIdBJhWDDBGRmUlOBg4elMtPGzYA//yT8ZyFBdCggQSb9u2BatU4WNhEMcikYpAhIjJjKSnA8eNylmbbNuDsWe3nfXyAdu0k1AQEAIUKqVIm5R6DTCoGGSKiAuTmTRksvG0bsHs38ORJxnO2tkCzZhlna7gIn1FjkEnFIENEVEA9eSJNLbdtk9v169rPV6yYEWoaNeIsKCPDIJOKQYaIiKAowMWLGaHm4EHtRfhcXIBWrSTUtG0LeHioVysBYJBJxyBDREQviIwE/vhDQs2OHdqzoDQaoG7djLM1NWtyIT4VMMikYpAhIqJsJScDJ05IqNm69cXmlsWKyVmali1lwHDRourUWcAwyKRikCEioly5cydjwHBwMBAbq/18jRoSagIDgVdfBeztVSnT3DHIpGKQISKiPEtIAPbvl8tQwcHAX39pP29rCzRuLKGmZUsJOZaWqpRqbhhkUjHIEBGRzty7JysMBwfL7fZt7efd3OTyU1qw8fNTp04zwCCTikGGiIj0QlGAy5cl0OzaBezdC0RHa+9TurQEmpYtgebNJehQjjDIpGKQISIig3j6VFYZTgs2R47ItjQaDVCnTsbZmoYN5dIUZYpBJhWDDBERqeLxYyAkREJNcDBw4YL28/b2QJMmEmpatJC+UBxfk45BJhWDDBERGYU7dzJCza5dQESE9vMuLjILqkkToGlToFYtwMpKnVqNAINMKgYZIiIyOooCnD+fEWwOHJAzOM9ydJTWCU2byq1OHcDGRp16VcAgk4pBhoiIjN7Tp0BoqEz1DgmRYPPokfY+9vYyribtjE39+oCdnSrlGgKDTCoGGSIiMjkpKcDZsxJqQkIk4Dx4oL2Pra2EmbQzNg0aAIUKqVOvHjDIpGKQISIik6coMlg4LdSEhLw4xsbKSnpEpQWbRo0AJyd16tUBBplUDDJERGR2FAW4ciXjjE1IyIuL81layoDhJk1k9WF/f8DTU51684BBJhWDDBERmT1FAcLDtYPN9esv7lemjIyzSbtVrmy0U74ZZFIxyBARUYF086Zchtq/Hzh8WGZJPc/JScbZpAWb+vUBV1eDl5oZBplUDDJERESQWVDHjsmKw4cPA0ePAjEx2vtoNHKWxt8/I9yULSvbDYxBJhWDDBERUSaSk4Fz5yTUpIWba9de3M/dXftyVJ06gIOD3stjkEnFIENERJRD9+5lhJojR6R3VEKC9j5WVkCNGtrhxtdX56UwyKRikCEiIsqjxETg9GkJNmm3O3de3O/LL4Fx43T61jn9/V1wmzgQERFR9mxsZABw/frAyJEyO+rWLe1gExoKVK+uWokMMkRERJQzGg1QooTcevaUbbGxqja3ZJAhIiKivFO5LYKFqu9ORERElA8MMkRERGSyGGSIiIjIZDHIEBERkclikCEiIiKTxSBDREREJkvVILN//3506NAB3t7e0Gg02LRpk9bziqJgwoQJKFasGOzt7REYGIgrV66oUywREREZHVWDTGxsLKpXr4758+dn+vz06dMxd+5cLFq0CMeOHUOhQoXQunVrxMfHG7hSIiIiMkaqLojXtm1btG3bNtPnFEXBnDlz8Nlnn6FTp04AgJUrV8LT0xObNm1Cz7QVBYmIiKjAMtoxMuHh4YiIiEBgYGD6NhcXF9SvXx9HjhzJ8nUJCQmIjo7WuhEREZF5MtogExERAQDw9PTU2u7p6Zn+XGaCgoLg4uKSfvPVQ2txIiIiMg5GG2TyauzYsYiKikq/3bp1S+2SiIiISE+MNsh4eXkBAO7du6e1/d69e+nPZcbW1hbOzs5aNyIiIjJPRtv92s/PD15eXti9ezdq1KgBAIiOjsaxY8cwZMiQHB9HUZT01xIREZFpSPu9nfZ7PCuqBpmYmBhcvXo1/XF4eDhCQ0Ph5uaGEiVKYMSIEfjiiy9QtmxZ+Pn5Yfz48fD29kbnzp1z/B6PHz8GAI6VISIiMkGPHz+Gi4tLls9rlJdFHT3at28fmjdv/sL2Pn36YMWKFVAUBRMnTsSSJUsQGRmJxo0bY8GCBShXrlyO3yMlJQV37tyBk5MTNBqNzmqPjo6Gr68vbt26VSAuXxWkz8vPar4K0uflZzVfBeXzKoqCx48fw9vbGxYWWY+EUTXImLLo6Gi4uLggKirKrP8ipSlIn5ef1XwVpM/Lz2q+CtrnfRmjHexLRERE9DIMMkRERGSyGGTyyNbWFhMnToStra3apRhEQfq8/KzmqyB9Xn5W81XQPu/LcIwMERERmSyekSEiIiKTxSBDREREJotBhoiIiEwWgwwRERGZLAaZbMyfPx+lSpWCnZ0d6tevjz///DPb/devX48KFSrAzs4OVatWxfbt2w1Uaf4EBQWhbt26cHJygoeHBzp37oywsLBsX7NixQpoNBqtm52dnYEqzrtJkya9UHeFChWyfY2pfq8AUKpUqRc+r0ajwdChQzPd35S+1/3796NDhw7w9vaGRqPBpk2btJ5XFAUTJkxAsWLFYG9vj8DAQFy5cuWlx83tz70hZPdZk5KSMHr0aFStWhWFChWCt7c33n77bdy5cyfbY+blZ8FQXvbd9u3b94Xa27Rp89Ljmtp3CyDTn1+NRoMZM2ZkeUxj/m71gUEmCz///DM+/PBDTJw4EadOnUL16tXRunVr3L9/P9P9Dx8+jDfffBP9+/fH6dOn0blzZ3Tu3Bnnzp0zcOW5FxISgqFDh+Lo0aMIDg5GUlISWrVqhdjY2Gxf5+zsjLt376bfbty4YaCK86dy5cpadR88eDDLfU35ewWA48ePa33W4OBgAMDrr7+e5WtM5XuNjY1F9erVMX/+/Eyfnz59OubOnYtFixbh2LFjKFSoEFq3bo34+Pgsj5nbn3tDye6zxsXF4dSpUxg/fjxOnTqFDRs2ICwsDB07dnzpcXPzs2BIL/tuAaBNmzZata9ZsybbY5ridwtA6zPevXsXy5Ytg0ajQbdu3bI9rrF+t3qhUKbq1aunDB06NP1xcnKy4u3trQQFBWW6f48ePZT27dtrbatfv77y7rvv6rVOfbh//74CQAkJCclyn+XLlysuLi6GK0pHJk6cqFSvXj3H+5vT96ooivLBBx8oZcqUUVJSUjJ93lS/VwDKxo0b0x+npKQoXl5eyowZM9K3RUZGKra2tsqaNWuyPE5uf+7V8Pxnzcyff/6pAFBu3LiR5T65/VlQS2aft0+fPkqnTp1ydRxz+W47deqktGjRItt9TOW71RWekclEYmIiTp48icDAwPRtFhYWCAwMxJEjRzJ9zZEjR7T2B4DWrVtnub8xi4qKAgC4ubllu19MTAxKliwJX19fdOrUCefPnzdEefl25coVeHt7o3Tp0ujduzdu3ryZ5b7m9L0mJibip59+Qr9+/bJtoGqq3+uzwsPDERERofXdubi4oH79+ll+d3n5uTdWUVFR0Gg0cHV1zXa/3PwsGJt9+/bBw8MD5cuXx5AhQ/Dw4cMs9zWX7/bevXvYtm0b+vfv/9J9Tfm7zS0GmUw8ePAAycnJ8PT01Nru6emJiIiITF8TERGRq/2NVUpKCkaMGIFGjRqhSpUqWe5Xvnx5LFu2DJs3b8ZPP/2ElJQUNGzYELdv3zZgtblXv359rFixAr///jsWLlyI8PBwvPrqq3j8+HGm+5vL9woAmzZtQmRkJPr27ZvlPqb6vT4v7fvJzXeXl597YxQfH4/Ro0fjzTffzLahYG5/FoxJmzZtsHLlSuzevRvTpk1DSEgI2rZti+Tk5Ez3N5fv9ocffoCTkxO6du2a7X6m/N3mhZXaBZBxGTp0KM6dO/fS66n+/v7w9/dPf9ywYUNUrFgRixcvxpQpU/RdZp61bds2/X61atVQv359lCxZEuvWrcvRv3JM2dKlS9G2bVt4e3tnuY+pfq8kkpKS0KNHDyiKgoULF2a7ryn/LPTs2TP9ftWqVVGtWjWUKVMG+/btQ0BAgIqV6deyZcvQu3fvlw7AN+XvNi94RiYTRYoUgaWlJe7du6e1/d69e/Dy8sr0NV5eXrna3xgNGzYMW7duxd69e+Hj45Or11pbW6NmzZq4evWqnqrTD1dXV5QrVy7Lus3hewWAGzduYNeuXRgwYECuXmeq32va95Ob7y4vP/fGJC3E3LhxA8HBwdmejcnMy34WjFnp0qVRpEiRLGs39e8WAA4cOICwsLBc/wwDpv3d5gSDTCZsbGxQu3Zt7N69O31bSkoKdu/erfWv1Wf5+/tr7Q8AwcHBWe5vTBRFwbBhw7Bx40bs2bMHfn5+uT5GcnIyzp49i2LFiumhQv2JiYnBtWvXsqzblL/XZy1fvhweHh5o3759rl5nqt+rn58fvLy8tL676OhoHDt2LMvvLi8/98YiLcRcuXIFu3btgru7e66P8bKfBWN2+/ZtPHz4MMvaTfm7TbN06VLUrl0b1atXz/VrTfm7zRG1Rxsbq7Vr1yq2trbKihUrlAsXLiiDBg1SXF1dlYiICEVRFOV///ufMmbMmPT9Dx06pFhZWSkzZ85ULl68qEycOFGxtrZWzp49q9ZHyLEhQ4YoLi4uyr59+5S7d++m3+Li4tL3ef7zfv7558rOnTuVa9euKSdPnlR69uyp2NnZKefPn1fjI+TYqFGjlH379inh4eHKoUOHlMDAQKVIkSLK/fv3FUUxr+81TXJyslKiRAll9OjRLzxnyt/r48ePldOnTyunT59WACizZ89WTp8+nT5TZ+rUqYqrq6uyefNm5cyZM0qnTp0UPz8/5cmTJ+nHaNGihfLtt9+mP37Zz71asvusiYmJSseOHRUfHx8lNDRU62c4ISEh/RjPf9aX/SyoKbvP+/jxY+Wjjz5Sjhw5ooSHhyu7du1SatWqpZQtW1aJj49PP4Y5fLdpoqKiFAcHB2XhwoWZHsOUvlt9YJDJxrfffquUKFFCsbGxUerVq6ccPXo0/bmmTZsqffr00dp/3bp1Srly5RQbGxulcuXKyrZt2wxccd4AyPS2fPny9H2e/7wjRoxI/7Px9PRU2rVrp5w6dcrwxefSG2+8oRQrVkyxsbFRihcvrrzxxhvK1atX0583p+81zc6dOxUASlhY2AvPmfL3unfv3kz/3qZ9npSUFGX8+PGKp6enYmtrqwQEBLzwZ1CyZEll4sSJWtuy+7lXS3afNTw8PMuf4b1796Yf4/nP+rKfBTVl93nj4uKUVq1aKUWLFlWsra2VkiVLKgMHDnwhkJjDd5tm8eLFir29vRIZGZnpMUzpu9UHjaIoil5P+RARERHpCcfIEBERkclikCEiIiKTxSBDREREJotBhoiIiEwWgwwRERGZLAYZIiIiMlkMMkRERGSyGGSIiIjIZDHIEFGBs2/fPmg0GkRGRqpdChHlE4MMERERmSwGGSIiIjJZDDJEZHApKSkICgqCn58f7O3tUb16dfzyyy8AMi77bNu2DdWqVYOdnR0aNGiAc+fOaR3j119/ReXKlWFra4tSpUph1qxZWs8nJCRg9OjR8PX1ha2tLV555RUsXbpUa5+TJ0+iTp06cHBwQMOGDREWFqbfD05EOscgQ0QGFxQUhJUrV2LRokU4f/48Ro4cibfeegshISHp+3z88ceYNWsWjh8/jqJFi6JDhw5ISkoCIAGkR48e6NmzJ86ePYtJkyZh/PjxWLFiRfrr3377baxZswZz587FxYsXsXjxYjg6OmrV8emnn2LWrFk4ceIErKys0K9fP4N8fiLSHXa/JiKDSkhIgJubG3bt2gV/f//07QMGDEBcXBwGDRqE5s2bY+3atXjjjTcAAP/99x98fHywYsUK9OjRA71798a///6LP/74I/31n3zyCbZt24bz58/j8uXLKF++PIKDgxEYGPhCDfv27UPz5s2xa9cuBAQEAAC2b9+O9u3b48mTJ7Czs9PznwIR6QrPyBCRQV29ehVxcXFo2bIlHB0d028rV67EtWvX0vd7NuS4ubmhfPnyuHjxIgDg4sWLaNSokdZxGzVqhCtXriA5ORmhoaGwtLRE06ZNs62lWrVq6feLFSsGALh//36+PyMRGY6V2gUQUcESExMDANi2bRuKFy+u9Zytra1WmMkre3v7HO1nbW2dfl+j0QCQ8TtEZDp4RoaIDKpSpUqwtbXFzZs38corr2jdfH190/c7evRo+v1Hjx7h8uXLqFixIgCgYsWKOHTokNZxDx06hHLlysHS0hJVq1ZFSkqK1pgbIjJPPCNDRAbl5OSEjz76CCNHjkRKSgoaN26MqKgoHDp0CM7OzihZsiQAYPLkyXB3d4enpyc+/fRTFClSBJ07dwYAjBo1CnXr1sWUKVPwxhtv4MiRI5g3bx4WLFgAAChVqhT69OmDfv36Ye7cuahevTpu3LiB+/fvo0ePHmp9dCLSAwYZIjK4KVOmoGjRoggKCsLff/8NV1dX1KpVC+PGjUu/tDN16lR88MEHuHLlCmrUqIEtW7bAxsYGAFCrVi2sW7cOEyZMwJQpU1CsWDFMnjwZffv2TX+PhQsXYty4cXjvvffw8OFDlChRAuPGjVPj4xKRHnHWEhEZlbQZRY8ePYKrq6va5RCRkeMYGSIiIjJZDDJERERksnhpiYiIiEwWz8gQERGRyWKQISIiIpPFIENEREQmi0GGiIiITBaDDBEREZksBhkiIiIyWQwyREREZLIYZIiIiMhk/R+8mOlmlnVMZwAAAABJRU5ErkJggg==","text/plain":["<Figure size 640x480 with 1 Axes>"]},"metadata":{},"output_type":"display_data"}],"source":["plt.plot(train_cost, 'r-', label='Train')\n","plt.plot(test_cost, 'b-', label='Test')\n","plt.xlabel('epoch'); plt.ylabel('cost'); plt.legend();"]},{"cell_type":"markdown","metadata":{},"source":["# Save and Load Model"]},{"cell_type":"code","execution_count":22,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T07:46:56.176044Z","iopub.status.busy":"2024-07-13T07:46:56.175162Z","iopub.status.idle":"2024-07-13T07:46:56.180047Z","shell.execute_reply":"2024-07-13T07:46:56.179154Z","shell.execute_reply.started":"2024-07-13T07:46:56.176009Z"},"trusted":true},"outputs":[],"source":["import os\n","os.makedirs('model', exist_ok=True)"]},{"cell_type":"code","execution_count":23,"metadata":{"execution":{"iopub.execute_input":"2024-07-13T07:46:56.765399Z","iopub.status.busy":"2024-07-13T07:46:56.764777Z","iopub.status.idle":"2024-07-13T07:46:56.815717Z","shell.execute_reply":"2024-07-13T07:46:56.814847Z","shell.execute_reply.started":"2024-07-13T07:46:56.765367Z"},"trusted":true},"outputs":[],"source":["# Save Model\n","torch.save(model.state_dict(), 'model/weights.pth')"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Load Model\n","model = NeuralNetLM(vocab, m = 60, n = 5, h = 50)\n","weights = torch.load('./model/weights.pth', map_location='cpu')\n","model.load_state_dict(weights)\n","model = model.to(device)"]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"datasetId":2058,"sourceId":131078,"sourceType":"datasetVersion"},{"isSourceIdPinned":true,"modelInstanceId":64569,"sourceId":76810,"sourceType":"modelInstanceVersion"}],"dockerImageVersionId":30746,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"}},"nbformat":4,"nbformat_minor":4}