ko_lm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# command allowing notebooks to execute python scripts in project\n",
    "import sys, os\n",
    "sys.path.insert(0, os.path.dirname(os.path.abspath(\".\")))\n",
    "\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import *\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import json\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score\n",
    "\n",
    "#from konlpy.tag import Okt\n",
    "#import jpype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 - Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1-1 Get wiki data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multifit.nlputils import split_wiki, get_wiki\n",
    "from multifit.nlputils2 import clean_files, get_num_tokens, get_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang = 'ko'\n",
    "data_path = Path('/home/amaleelhamri/multifit/data/korean')\n",
    "name = f'{lang}wiki'\n",
    "path = data_path/name\n",
    "path.mkdir(exist_ok=True, parents=True)\n",
    "mdl_path = path/'models'\n",
    "bs = 256\n",
    "\n",
    "lm_fns3 = [f'{lang}_wt_sp15_multifit', f'{lang}_wt_vocab_sp15_multifit']\n",
    "lm_fns3_bwd = [f'{lang}_wt_sp15_multifit_bwd', f'{lang}_wt_vocab_sp15_multifit_bwd']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "downloading...\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unzipping...\n",
      "extracting...\n",
      "CPU times: user 2min 5s, sys: 4.14 s, total: 2min 9s\n",
      "Wall time: 7min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "get_wiki(path,lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n",
      "800000\n",
      "900000\n",
      "1000000\n",
      "1100000\n",
      "1200000\n",
      "1300000\n",
      "1400000\n",
      "1500000\n",
      "CPU times: user 4.54 s, sys: 1.66 s, total: 6.2 s\n",
      "Wall time: 6.24 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dest = split_wiki(path, lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dest = path/\"docs\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<doc id=\"5\" url=\"https://ko.wikipedia.org/wiki?curid=5\" title=\"지미 카터\">\r\n",
      "지미 카터\r\n",
      "\r\n",
      "제임스 얼 \"지미\" 카터 주니어(, 1924년 10월 1일 ~ )는 민주당 출신 미국 39번째 대통령 (1977년 ~ 1981년)이다.\r\n"
     ]
    }
   ],
   "source": [
    "!head -n4 {path}/{name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = \"docs\"\n",
    "clean_files(path, folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/amaleelhamri/multifit/data/korean/kowiki/docs/섀도 복사본.txt'),\n",
       " PosixPath('/home/amaleelhamri/multifit/data/korean/kowiki/docs/디도 (성경).txt'),\n",
       " PosixPath('/home/amaleelhamri/multifit/data/korean/kowiki/docs/개그야.txt'),\n",
       " PosixPath('/home/amaleelhamri/multifit/data/korean/kowiki/docs/펠릭스 멘델스존.txt'),\n",
       " PosixPath('/home/amaleelhamri/multifit/data/korean/kowiki/docs/듀얼쇼크.txt')]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dest = path/'docs'\n",
    "dest.ls()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29814 files - 29929080 tokens\n",
      "CPU times: user 4.02 s, sys: 736 ms, total: 4.76 s\n",
      "Wall time: 4.75 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "num_files, num_tokens = get_num_tokens(dest)\n",
    "print(f'{num_files} files - {num_tokens} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As there are less than 100 000 000 tokens in the initial corpus, we use it.\n",
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 97.8 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "path_corpus = get_corpus(dest, path, num_tokens, obj_tokens=1e8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 - 2 Build databunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "OKT_TOKENIZER_KO = Okt()\n",
    "def tokenize_korean_text(text):\n",
    "    \"\"\"\n",
    "    Function that tokenizes and normalizes korean texts\n",
    "    :param str text: text written in korean\n",
    "\n",
    "    :return: list of korean tokens\n",
    "    :rtype: list(str)\n",
    "    \"\"\"\n",
    "    if jpype.isJVMStarted():\n",
    "        jpype.attachThreadToJVM()\n",
    "    return OKT_TOKENIZER_KO.morphs(str(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20min 11s, sys: 6.11 s, total: 20min 17s\n",
      "Wall time: 5min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data = (TextList.from_folder(dest, processor=[OpenFileProcessor(), SPProcessor(max_vocab_sz=10000, lang='ko')])\n",
    "        .split_by_rand_pct(0.1, seed=42)\n",
    "        .label_for_lm()\n",
    "        .databunch(bs=bs, num_workers=8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.save(f'{path}/{lang}_databunch_sp15_multifit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 26833)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.vocab.itos), len(data.train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text ▁xxbos ▁디 도 ( 공 동 번 역 , ▁개신교 ), ▁티 토 ( 가 톨 릭 ) ( ; ▁ ) 는 ▁초기 ▁기독교 의 ▁선 교 사 이자 ▁교회 ▁지도자 로 , ▁디 도 서 ▁등의 ▁바 울 ▁서 신 에서 ▁언급 되는 ▁사 도 ▁바 울 로 의 ▁동 역 자 이다 . ▁바 울 로 에 ▁의해 ▁기독교 로 ▁개 종 한 ▁것으로 ▁보 이며 , ▁전 승 에 ▁의하면 ▁그는 ▁크레 타 의 ▁감독으로 ▁세워 졌다 고 ▁한다 . ▁디 도 는 ▁코 린 토스 ( 고 린 도 ) ▁교회 에 ▁예 루 살 렘 의 ▁ 가 난 한 ▁성 도 들을 ▁도와 달 라는 ▁편지 를 ▁바 울 로 에게서 ▁받아 ▁전달 한다 . ▁이후에 ▁크레 타 에서 ▁디 도 는 ▁죽 을 때 ▁ 까지 ▁크레 타 섬 의 ▁모든 ▁도시 에 ▁장 로 를 ▁임명 했고 , ▁칸 디아 ( 현재 의 ▁ 이라 클리 오 ) ▁주변 에 ▁있는 ▁도시 인 ▁고 르 티 나 에서 ▁숨 을 ▁거두었다 고 ▁전 해 진다 . ▁디 도 는 ▁헬 라 인 이며 ▁안 티 오 키 아 ▁출신 으로 ▁여겨 지 는데 , ▁젊은 ▁나이 에 ▁그리스 ▁철학 과 ▁시 학 을 ▁배 웠다 고 ▁전 해 진다 . ▁그는 ▁또한 ▁바 울 로 에 ▁의해 ▁회 심 한 것 으로 ▁보 이며 , ▁바 울 로 의 ▁통 역 자 이자 ▁비 서 관으로 ▁활동 했다 . ▁그는 ▁또한 ▁ 49 세에 ▁예 루 살 렘 에서 ▁열린 ▁모 세의 ▁ 율 법 에 ▁대한 ▁공 회 에 ▁바 울 로 , ▁ 바르 나 바 와 ▁함께 ▁동 행 했고 ( 갈 ▁2 : 1 -3 ) . ▁바 울 로 가 ▁비록 ▁유대인 에 ▁대한 ▁사 역 을 ▁위해 ▁디 모 테 오 의 ▁할 례 에 ▁동 의 했지만 , ▁기본적으로 ▁이 방 인 ▁개 종 자에게 ▁할 례 가 ▁필요 하지 ▁않다 는 ▁태도 를 ▁취 했 으므로 ▁디 도 는 ▁할 례 를 ▁받지 ▁않았다 ( 갈 ▁2 : 3 - 5 ) . ▁바 울 로 가 ▁ 56 세 쯤 ▁되어 ▁아시아 에서 ▁떠나 게 ▁ 되었을 ▁때 ▁디 도 를 ▁ 에 페 소 스 에서 ▁코 린 토스 로 ▁보내 며 , ▁디 모 테 오 의 ▁고 린 도 전 서 ▁전달 ( 고 전 ▁16 : 10 - 11 ) 과 ▁바 울 로 의 ▁방문 ( 고 후 ▁2 : 1 , 5 - 11 ) 으로 ▁인해 ▁발생한 ▁안 좋 은 ▁ 일 들을 ▁치 유 하는 ▁임무 를 ▁맡 긴 다 . ▁바 울 로 의 ▁여 정 동안 ▁디 도 는 ▁엄 중 한 ▁편지 ( se ve re ▁xxmaj ▁le t ter ) 를 ▁전달 하였고 ( 고 후 7 : 8 ), ▁이에 ▁대한 ▁코 린 토스 ▁교회 의 ▁부정 적 ▁반응 을 ▁ 듣 기 ▁위해 ▁ 트로 아 스 ( 드로 아 ) 에서 ▁디 도 를 ▁만 난 ▁것으로 ▁보인다 ( 고 후 2 : 12 - 13 ) . ▁이 ▁임무 를 ▁성공 적으로 ▁끝 마 친 ▁이후 ▁디 도 는 ▁북쪽 으로 ▁가서 ▁마 케 도 니아 에 ▁있는 ▁바 울 로 를 ▁만 난 다 . ▁거 기 서 ▁바 울 로 는 ▁디 도 의 ▁성공 ( 고 후 ▁7 : 6 - 15 ) 을 ▁매우 ▁기 뻐 하며 ▁고 린 도 후 서를 ▁쓰 게 ▁된 ▁것이다 . ▁디 도 는 ▁이번 에는 ▁여러 명의 ▁동 행 자와 ▁함께 ▁고 린 도 후 서를 ▁전달 하게 ▁된다 . ▁바 울 로 는 ▁후에 ▁코 린 토스 에 ▁있는 ▁디 도 와 ▁합류 한 ▁뒤 , ▁디 도 를 ▁통해 ▁예 루 살 렘 에 ▁성 금을 ▁전달 하도록 ▁한다 . ▁즉 ▁디 도 는 ▁갈등 을 ▁중 재 하고 ▁관리 하는 일 과 ▁선 교 활동 에 ▁큰 ▁자 질 을 ▁보였다 . ▁초대 ▁교회 의 ▁전 승 에 ▁의하면 ▁바 울 로 는 ▁그의 ▁첫 번째 ▁로마 투 옥 에서 ▁풀 려 난 ▁뒤 ▁크레 타 섬 에 ▁들 려 ▁복 음을 ▁전 했다 . ▁그러나 ▁더 ▁많은 ▁교회 들이 ▁바 울 로 의 ▁방문 을 ▁필요 로 ▁ 했기 ▁때문에 , ▁디 도 를 ▁크레 타 섬 의 ▁감독으로 ▁임명 ( 딛 ▁1 : 5 ) 하여 ▁자신이 ▁못 다 한 ▁ 일 들을 ▁마 저 ▁ 하도록 ▁한 ▁뒤 ▁떠나 게 ▁된다 . ▁요한 네스 ▁크 리 소 스토 무 스는 ▁이로써 ▁바 울 로 가 ▁디 도 를 ▁크게 ▁신 임 했 음을 ▁알 ▁수 ▁있다고 ▁말한다 . ▁바 울 로 는 ▁후에 ▁이 피 로스 ▁지역에 ▁있는 ▁니 코 폴 리스 로 ▁디 도 를 ▁소 환 한다 ( 딛 ▁3 : 12 ) . ▁이후 ▁디 도 는 ▁달 마 티아 로 ▁여 정을 ▁떠나 게 ▁ 되는데 ( 딤 후 ▁4 : 10 ), ▁이후 ▁디 도 의 ▁죽음 에 ▁대해서는 ▁신 약 성 경 에서 ▁전 하지 ▁않는다 . ▁고 린 도 후 서 와 ▁갈라 디아 서 에서 ▁사용 된 ▁\" 디 도 \" 라는 ▁이름은 ▁디 모 테 오 가 ▁비 격 식 으로 ▁사용한 ▁이름 에 ▁불과 하다 는 ▁주장 이 ▁제기 되었는데 , ▁그 ▁주 된 ▁이유는 ▁디 모 테 오 와 ▁디 도 가 ▁바 울 로 와 ▁오랫동안 ▁ 굉 장 히 ▁ 가 깝 게 ▁지낸 ▁동 역 자 임 에도 ▁불구하고 ▁이 ▁두 ▁서 간 에서 ▁그 ▁이름이 ▁동시에 ▁등장 하지 ▁않는다 는 ▁것이다 . ▁이 ▁이론 은 ▁고전 ▁4 : 17 , ▁16 : 10 ; ▁고 후 ▁2 : 13 , ▁7 : 6 , ▁13 - 14 , ▁12 : 18 ; ▁행 ▁19 : 22 에 ▁등장하는 ▁한 ▁여 정 이 ▁디 도 이자 ▁디 모 데 인 ▁한 ▁사람의 ▁여 정 이라고 ▁주장한다 . ▁그러나 ▁디 모 데 후 서는 ▁디 도 가 ▁달 마 티아 로 ▁여 정을 ▁떠 났다 고 ( 딤 후 ▁4 : 10 ) ▁기록 하여 ▁이 ▁주장 을 ▁반 박 하고 ▁있다 . ▁바 울 로 가 ▁디 모 테 오 를 ▁할 례 했으나 ( 행 ▁16 : 3 ) ▁디 도 에게는 ▁하지 ▁않았다 는 ▁기록 ( 갈 ▁2 : 3 ) ▁역시 ▁이 ▁둘 이 ▁다른 ▁사람 이라는 ▁증거 가 ▁될 ▁수 ▁있으나 , ▁마 리우스 ▁빅 토리 누스 가 ▁주 석 에 ▁남 긴 것 ▁ 처럼 ▁갈라 디아 서 ▁2 장 ▁4 절 의 ▁일부 ▁사 본 은 ▁디 도 가 ▁할 례 를 ▁받았다 고 ▁말한다 . ▁디 도 의 ▁축 일 은 ▁트리 덴 티 노 ▁달 력 에는 ▁나와 있 지 ▁않았 지만 , ▁18 54 년에 ▁2 월 ▁6 일 로 ▁지정 되었다 . ▁1969 년에 ▁가톨릭 ▁교회 는 ▁성 ▁바 오 로 ▁사 도 의 ▁회 심 ▁축 일 ▁바로 ▁다음날 인 ▁1 월 ▁26 일 을 ▁바 울 로 의 ▁두 ▁제 자인 ▁디 모 테 오 와 ▁디 도 의 ▁축 일 로 ▁지정 했다 . ▁미국 ▁복 음 주의 ▁루 터 회 는 ▁디 모 테 오 와 ▁디 도 에 ▁더 해 ▁실 라 ▁역시 ▁같은 ▁날 에 ▁기념 한다 . ▁동 방 ▁정 교회 는 ▁8 월 ▁25 일 과 ▁1 월 ▁4 일 을 ▁축 일 로 ▁삼 는 다 . ▁그의 ▁성 물 은 ▁지금 은 ▁두 개 골 밖에 ▁남아있 지 ▁않은 데 , ▁터키 ▁점령 기 때 ▁베 네 치 아 로 ▁이 송 되었다 가 ▁1966 년에 ▁반 환 되어 ▁지금까지 ▁크레 타 ▁섬 의 ▁ 이라 클리 오 의 ▁성 ▁디 도 ▁교회 에 ▁있다 . < re f ▁ na me = \" the ▁xxmaj ▁ or t ho do x ▁xxmaj ▁m ess en g er , ▁v . ▁8 ( 7 ▁/ ▁8 ), ▁xxmaj ▁j ul y ▁/ ▁xxmaj ▁ au g ▁1997 \" > the ▁xxmaj ▁ or t ho do x ▁xxmaj ▁m ess en g er , ▁v . ▁8 ( 7 ▁/ ▁8 ), ▁xxmaj ▁j ul y ▁/ ▁xxmaj ▁ au g ▁1997 < ▁/ ▁re f > ▁성 ▁디 도 는 ▁미군 ▁군 종 ▁장교 의 ▁성인 이다 ."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train_ds.x[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 - Train LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_data(path, f'{lang}_databunch_sp15_multifit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = awd_lstm_lm_config.copy()\n",
    "config['qrnn'] = True\n",
    "config['n_hid'] = 1550 #default 1152\n",
    "config['n_layers'] = 4 #default 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "perplexity = Perplexity()\n",
    "learn = language_model_learner(data, AWD_LSTM, config=config, drop_mult=1., pretrained=False, \n",
    "                               metrics=[error_rate, accuracy, perplexity]).to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3gc1fn28e+jbsmW3OQmW+4VGzeBWyi2CaEk1JhQTDA1dEJI+CUX70sS8iYEQhLSKKaYav8CmCQQOoQWcEE24IYx7pZ7lS3JVj3vHzuy10K2JWtnZ1d7f65rL+3OnJ15LK90a+bMnGPOOUREJHElBV2AiIgES0EgIpLgFAQiIglOQSAikuAUBCIiCS4l6AIaq3379q5Hjx5BlyEiElfmzZu3zTmXW9+6uAuCHj16UFhYGHQZIiJxxczWHGqdTg2JiCQ4BYGISIJTEIiIJDgFgYhIglMQiIgkOAWBiEiCUxCIiCQ4X+8jMLNbgKsBAx5xzt1fZ/0lwP94L0uA65xzn/tRy5eb9vDKgg2Ne5PZgadH9zbvvYd+d922B7/vCNut5821i2r3eeD1odfVbqu2jdW+9p7jrUsKW7b/uRlJduB1+Nckb11ykpGUZCSbhZ6bkZLsfU0KLUtJrn2eRGqykZqc5D1Cz9OSk0hKasz/gog0lG9BYGaDCYXA8UAF8LqZveKc+yqs2SrgJOfcTjM7HZgKjPKjnuVbSvjLu8sPWubcoX8Ra5qG2JOSZGSkJpORmkR6SjIt0pLJSksmMy2FrPRkWmWkktMi9GidmUrH7Aw652TQOacFua3SSVaQiNTLzyOCgcBs51wZgJm9D5wL3FvbwDn3cVj72UBXv4o589jOnHnsmX5tfr+6E/0cLlAOlzVf287X1oevc/Xuq/a1w4U9P3j7rradO9DOeetr17nQSmrC2tQ4r623rLrG7X9fdU1oWU1NqF11jaPaOWpqwp9DVU1N6LX3qKxxVFXXUFXjqKp2VFbXUFldQ3lVDRVVNVRU11BeWcO+qmr2VYYeZRXVlJVXs2HXPkrKS9hVVsHufVVf+36mpSQxoFMrjumSzaAuOXyjT3t6ts86zP+ASOLwMwgWAb82s3bAXuAM4HBjQ1wJvFbfCjO7BrgGID8/P8JlRlbd0zWHO/VzhC01uZZEVV3jKN5byabifWzavZeNxftYva2UJRt389qiTcyYuw6A4fmtOW9EV75zbGdaZ6YFXLVIcMzPqSrN7ErgBkLn/5cAe51zt9bTbjzwAPAN59z2w22zoKDAaawhOVrOOYp27uXVhRuZOb+IZZtLSEtJ4vZv9eeKcT3VDyHNlpnNc84V1LsuWnMWm9lvgCLn3AN1lh8L/AM43Tm37EjbURBIpDjnWLxhN/e//RVvf7GZk/rlct+koeS2Sg+6NJGIO1wQ+Hr5qJl18L7mA+cBM+qszwdeBC5tSAiIRJKZMTgvh0e+P5JfnTOY2Su3c/qfPuCDZVuDLk0kqvy+j2CmmS0BXgZu8K4OutbMrvXW3wm0Ax4ws8/MTH/qS9SZGZeO7s5LN36DtllpXPN0Icu3lARdlkjURO3UUKTo1JD4acvufZx6/wfkt81k5nVjSU3WPZfSPAR2akgk3nTIzuDuc4ewoKiYv/xn+ZHfINIMKAhE6jh9SGfOG5HH395dzvy1O4MuR8R3CgKRevzirGPolJ3Bj/7+GWUVX79BTaQ5URCI1CM7I5XfXzCUNTvK+KtOEUkzpyAQOYTRvdpxxpDOPDVrDcVllUGXI+IbBYHIYdxwch9Kyqt4ctbqoEsR8Y2CQOQwBnXJZuKADjz+0SpKy9VXIM2TgkDkCG6Y0IddZZVMn7M26FJEfKEgEDmCEfltGNenHVM/XMm+yuqgyxGJOAWBSAPcML4PW/eU8/y8oqBLEYk4BYFIA4zp1Y4R+a156L0VVFbXBF2OSEQpCEQawMy4/uQ+rN+1lzcWbwq6HJGIUhCINNCEAR3o3i6TaR+tDroUkYhSEIg0UFKScdmYHsxbs5MFRbuCLkckYhQEIo0wqaArWWnJPKGjAmlGFAQijdAqI5VJBd14ecEGtuzZF3Q5IhGhIBBppO+P6U5ltWPGnHVBlyISEQoCkUbqlduS8f1zeWbOGiqqdCmpxD8FgchRmDKuJ1v3lPPqwo1BlyLSZAoCkaNwYt/29M7N4omPVwddikiTKQhEjoKZccmo7ny2bheLNxQHXY5IkygIRI7S+SO6kp6SpFFJJe4pCESOUk5mKt8+tgv//HQ9JZqrQOKYgkCkCS4elU9pRTUvfbYh6FK+5ouNu9lRWhF0GRIHFAQiTTAivzUDOrXi2TlrcM4FXc5Bpkyby8WPzGZvheZQkMNTEIg0QajTOJ/FG3azoCi2Oo13llaydNMe7vjHwpgLKYktCgKRJjpneB6Zack8O2dN0KXsV1FVQ0V1DZ1zMnjx0/VMn6sObTk0BYFIE7XKSOWsoV14+fONFO+tDLocAMoqQp3XV36jJyf1y+WXLy3h83UaMVXqpyAQiYBLRnVnb2U1M2NkKsvaq5iyM1K5/3vDyG2VzvXPzo+ZoJLYoiAQiYAhXXMYkd+ap2atpqYm+PPxZV4HcWZ6Mm2y0vjrxcPZULyXB99bEXBlEosUBCIRMmVcT1ZvL+P9ZVuDLmX/EUFWegoAw/PbcPbQLkz7aBWbijV8thxMQSASIacP7kTH7HSmxcD4Q2XloSOCrLSU/ctuO7U/Nc5x/9vLgipLYpSCQCRCUpOTmDyqOx8s28ryLSWB1nLgiCB5/7JubTO5ZFR3nitcF3h9Elt8DQIzu8XMFpnZYjP7YT3rzcz+bGbLzWyBmY3wsx4Rv108Kp+0lCSeDPiooLQ2CMKOCABumtCHzLQU7nvjyyDKkhjlWxCY2WDgauB4YCjwbTPrW6fZ6UBf73EN8KBf9YhEQ7uW6Zw1tAsz5xcFeoVO7eWjtX0Etdq1TOfqE3rx+uJNzF+7M4jSJAb5eUQwEJjtnCtzzlUB7wPn1mlzNvCUC5kNtDazzj7WJOK7KWN7UFZRzfOFwU1lWeL1EbSsEwQAV53Qk/Yt0/j5vxZrsDwB/A2CRcCJZtbOzDKBM4BuddrkAeE/LUXesoOY2TVmVmhmhVu3Bn9FhsjhDM7L4fgebXly1mqqqoOZyrKsoookg4zUr/+IZ6Wn8Jtzh7Bk426+/9gc9uzTvQWJzrcgcM59AdwDvAW8DnwO1P3zw+p7az3bmuqcK3DOFeTm5ka8VpFIu+IbPVm3Yy+vLdoUyP5LyqvISkvBrL4fMTj1mE787eLhLCgq5vuPz2W3wiCh+dpZ7Jx7zDk3wjl3IrAD+KpOkyIOPkroCsTeeL4ijXTqoI70ys3iwfdWBDLgW1l5NZlhVwzV57TBnXngkhEsWl/MpY/O0V3HCczvq4Y6eF/zgfOAGXWavAR837t6aDRQ7JzTbOAS95KSjGtP6s2Sjbv54KttUd9/SUXV1zqK63PqMZ148JKRLN6wm1+/siQKlUks8vs+gplmtgR4GbjBObfTzK41s2u99a8CK4HlwCPA9T7XIxI15wzLo1N2Bg++tzzq+y7zTg01xCmDOjJlbA+en1fEFxt3+1yZxCK/Tw2d4Jwb5Jwb6px7x1v2kHPuIe+5c87d4Jzr7Zwb4pwr9LMekWhKS0niqhN6MnvljqhfqllaXn3QzWRHcuOEPmRnpHL3a0t9rEpile4sFvHRRcfnk9MilYeiPNhbSSOOCABaZ6Zx04Q+fLBsa0yMlSTRpSAQ8VFWegqXje3Bm0s2s3zLnqjtt6yBfQThLh3Tnfy2mdz96hdUx8AIqhI9CgIRn00Z24MWqck89P7KqO2zpLy60UGQnpLM/5w2gKWb9jBzfmzMqyDRoSAQ8VnbrDQuKOjKvz5bz+bd0RkCuqyiiqy0hvcR1DpjSCeG57fm929+uX+YCmn+FAQiUXDlN3pRXeOY9tFq3/dVU+Moq2j8EQGAmfF/zhzI5t3lPPrhKh+qk1ikIBCJgvx2mZw+uDPPzlnj+/g+ZZXeXASNuGoo3MjubTntmE48/P4Ktu4pj2RpEqMUBCJRcvWJvdizr4r/nbvW1/2Ultc/8mhj3H5af8qrajSJTYJQEIhEybBurTm+Z1se/+8qKn0cjK7kEHMRNEav3JZcPCqf//1Ek9gkAgWBSBT94MRebCjex6sL/RtJZf80lU04IgC4ZWJfWqQmc8/rusmsuVMQiETR+P4d6J2bxcPvr/RtMLoDRwRH10dQq13LdK47uTdvLdnM3FU7IlGaxCgFgUgUJSUZ15zYiyUbdzNr5XZf9nGo2cmOxhXjetIpO4Nfv7KEGt1k1mwpCESi7OxhebTOTOWZ2Wt82X5JBDqLa7VIS+bH3+rP50XFvLxAI8Q3VwoCkSjLSE3mgoJuvLF4sy83mJVVNO3y0brOG57HMV2yuff1L9nnXZoqzYuCQCQAl4zKp8Y5ps+J/KWkkbh8NFxSknHHmQNZv2svj3+km8yaIwWBSAC6t8vipH65zJi7NuKXkpZ6Vw1lpkbmiABgbO/2nDKwAw+8u4JtJbrJrLlREIgE5PtjurNlTzlvLdkc0e2WVlSRkZpESnJkf7x/evpA9lZW6yazZkhBIBKQk/p1oGubFjw1a3VEt9vYuQgaqk+HllwyKp8Zc9dFdUht8Z+CQCQgyUnG5NHdmb1yB19tjtwv1rLyxs9F0FC3TOxLZmoy97z+pS/bl2AoCEQCdEFBN9JSkng6gpeSlpRXk9nEm8kOpV3LdK7VTWbNjoJAJEBts9L49pDOvDh/fcRGJS2rqKKlT0cEELrJrGN2One/9oVvd0dLdCkIRAJ26ZjulJRX8c9P10dke6U+nhqC0E1mt32zP5+u3cXrizb5th+JHgWBSMCGdWvN4Lxsnpm9JiJ/YZdWVEfsZrJDOX9kV/p1bMm9b3zp60iqEh0KApGAmRmXju7O0k17KFyzs8nbK/XpqqFwyUnGT08fwKptpb7PryD+UxCIxICzhubRKiOFp2c1vdO4xOdTQ7XG9+/A6F5t+dM7X+2/m1nik4JAJAa0SEtm0shuvLZoY5Omh3Sudr5if08NQehI5vbTBrCtpIInZ632fX/iHwWBSIyYPDqfymrH3z85+lMt5VU1VNc4Mn0+NVRrRH4bJg7owMPvr2T3vsqo7FMiT0EgEiN65bbkhL7tmT5nLVVH2QFbe4rGz8tH67r1m/0o3lvJYx9qQLp4pSAQiSGTR3dnQ/E+3lm65ajev3/AOZ9uKKvP4LwcTh/cicf+u4qdpRVR269EjoJAJIZMHNCBLjkZRz3+UGlF9I8IIHRUUFpRxcMfrIzqfiUyFAQiMSQlOYnJY7rz0fLtLDuK8YdqTw1lRjkI+nVsxdlDu/DEx6vYsifyk+2IvxQEIjHmwuPySU9J4omPVzf6vaXe7GQto3DVUF23nNKPymrHg++tiPq+pWkUBCIxpm1WGucMy+PF+UUUlzXuSpxIz07WGD3bZ3He8Dymz1nLFh+m4BT/+BoEZnarmS02s0VmNsPMMuqszzezd83sUzNbYGZn+FmPSLy4bGwP9lXW8PfCxl1Kun/i+ihdPlrXjRP6UFXj1FcQZ3wLAjPLA24GCpxzg4Fk4MI6zf4P8Jxzbri37gG/6hGJJ4O6ZDOqZ1uemrWG6pqGjz9UFuARAYSm4Dx3eB7PzF6jvoI44vepoRSghZmlAJnAhjrrHZDtPc+pZ71IwpoytgdFO/fyzhcNn8qyto8gmpeP1nXj+NBRwdT3dVQQL3wLAufceuA+YC2wESh2zr1Zp9kvgMlmVgS8CtzkVz0i8eabgzrSJSejUZ3GpeVVpCQZ6SnBdf/1aJ/FOcPyeGaOjgrihZ+nhtoAZwM9gS5AlplNrtPsIuAJ51xX4AzgaTP7Wk1mdo2ZFZpZ4datW/0qWSSmpCQncemYHny8YjuLNxQ36D2l5VVkpiVjZj5Xd3g3TuhDRVWNjgrihJ9/NpwCrHLObXXOVQIvAmPrtLkSeA7AOTcLyADa192Qc26qc67AOVeQm5vrY8kiseXiUfm0TE9p8CWZpRXVUb+ZrD4922dxzvDQUUFTBtGT6PAzCNYCo80s00J/nkwEvqinzUQAMxtIKAj0J7+IJ6dFKpeMzufVhRtZva30iO1Ly6uifjPZodw0oW/oqOAD3VcQ6/zsI5gDvADMBxZ6+5pqZneZ2Vles9uAq83sc2AGMMVpElSRg1z5jZ6kJCfxcAN+oUZrLoKG6On1FTw9ew3bSnRUEMt87VFyzv3cOTfAOTfYOXepc67cOXenc+4lb/0S59w459xQ59ywejqTRRJeh1YZTBrZlZnz1rP5CDdqlVVUB3JX8aHs7yvQfQUxrUFBYGa9zSzde36ymd1sZq39LU1Eav3gxN5U1dTw6IeH/4Ua6iyOjSMCCA2tfdbQLjw9S0cFsayhRwQzgWoz6wM8RuhKoOm+VSUiB8lvl8l3hnbh2Tlr2VV26KGeSyuqYqKzONyNE/pSXlXNIzoqiFkNDYIa51wVcC5wv3PuVqCzf2WJSF3Xndybsopqpn20+pBtSsurA72ZrD59OrTkO0O78NSsNWzXUUFMamgQVJrZRcBlwL+9Zan+lCQi9RnQKZszhnTiwfdXsHTT7nrblJbH3hEBhK4g2ldVzdQjnNqSYDQ0CC4HxgC/ds6tMrOewDP+lSUi9bnr7MFkZ6Ry84xP2VdZfdC6quoayqtqYqqPoFafDqG+gqc+Vl9BLGpQEHhX99zsnJvh3THcyjn3W59rE5E62rdM575Jx7Jscwm/fW3pQetqxxnKiqGrhsLdPDHUV/Dw+7qvINY09Kqh98ws28zaAp8D08zsD/6WJiL1Obl/By4f14MnPl7Nu2FzGwc5F0FD9M5tyTnD83hq1hrNVxBjGnpqKMc5txs4D5jmnBtJaAgJEQnA/5w2gAGdWvGTFz7ff6ol1oMA4OYJfamqcTygWcxiSkODIMXMOgMXcKCzWEQCkpGazJ8uHM6uskr+8s5XQLDTVDZUj/ZZfHdEV6bPWcvG4r1BlyOehgbBXcAbwArn3Cdm1gv4yr+yRORI+ndqxaSCrsyYu471u/YemLg+BjuLw904oQ8Ox9/eXR50KeJpaGfx8865Y51z13mvVzrnzve3NBE5khsn9AXgr/9Zvj8IYvHy0XDd2mZyQUE3/v7JOop2lgVdjtDwzuKuZvYPM9tiZpvNbKaZdfW7OBE5vLzWLbjw+G48X7iOLzbuAYKdnayhbhjfB8P409s6sRALGnpqaBrwEqEJZvKAl71lIhKwG8b3ITnJeMS7WSvWjwgAurRuweTR3Zk5v4jlW/YEXU7Ca2gQ5DrnpjnnqrzHE4BmiBGJAR2zM5g8ujsltX0EcRAEADeM702L1GR+/+ayoEtJeA0Ngm1mNtnMkr3HZGC7n4WJSMNdd3LolypAZmrsnxoCaNcynatO6MVrizaxoGhX0OUktIYGwRWELh3dRGgi+u8SGnZCRGJA+5bp3DihDwM6tSIpKdj5ihvjqhN60iYzld+98WXQpSS0hl41tNY5d5ZzLtc518E5dw6hm8tEJEbcML4Pr91yQtBlNEqrjFRuGN+HD7/axscrtgVdTsJqygxlP4pYFSISEaHpwePL5NHd6ZyTwb2vf4lmqg1GU4Ig/j5xIhJzMlKT+eEpffls3S7eWLwp6HISUlOCQNEtIhFx/oiu9OnQkntf/5Kq6pqgy0k4hw0CM9tjZrvreewhdE+BiEiTpSQncfu3+rNyWyl/L1wXdDkJ57BB4Jxr5ZzLrufRyjkXHxcri0hc+OagjhR0b8P9b39FWUVV0OUklKacGhIRiRgz42dnDGDrnnIe+3BV0OUkFAWBiMSMkd3bcuqgjjz8wUpNdB9FCgIRiSm3nzaAvZXV/PkdDUgXLQoCEYkpfTq05MLjuvHMnLV8tVkD0kWDgkBEYs6PvtmPrLRk7vr3Et1kFgUKAhGJOe1apvPDU/rx4VfbeOeLLUGX0+wpCEQkJl06pju9c7P4f68sobyqOuhymjUFgYjEpNTkJO78zjGs3l7GEx+tDrqcZk1BICIx66R+uUwc0IG//Gc5W/bsC7qcZktBICIx7Y4zB1JRXcOd/1ysjmOfKAhEJKb1ym3Jraf04/XFm3hl4cagy2mWfA0CM7vVzBab2SIzm2FmGfW0ucDMlnjtpvtZj4jEp6tP6MnQrjnc+a/FuuPYB74FgZnlATcDBc65wUAycGGdNn2BnwHjnHPHAD/0qx4RiV8pyUn8btJQSvZVcedLi4Mup9nx+9RQCtDCzFKATGBDnfVXA39zzu0EcM7pgmERqVe/jq245ZS+vLJgI6/qFFFE+RYEzrn1wH3AWkIT3hc7596s06wf0M/MPjKz2WZ2Wn3bMrNrzKzQzAq3bt3qV8kiEuN+cGIvhuTlcOe/FlFcVhl0Oc2Gn6eG2gBnAz0JTWKTZWaT6zRLAfoCJwMXAY+aWeu623LOTXXOFTjnCnJzc/0qWURiXEpyEr89fwg7Siu4942lQZfTbPh5augUYJVzbqtzrhJ4ERhbp00R8C/nXKVzbhXwJaFgEBGp1zFdcrh8XE+mz13Lp2t3Bl1Os+BnEKwFRptZppkZMBH4ok6bfwLjAcysPaFTRSt9rElEmoFbv9mPjq0yuOMfizTHcQT42UcwB3gBmA8s9PY11czuMrOzvGZvANvNbAnwLvAT59x2v2oSkeahZXoKd35nEEs27ubJWWuCLifuWbzdqVdQUOAKCwuDLkNEAuac4/InPuGTVTt457aT6ZTztduUJIyZzXPOFdS3TncWi0hcMjPuOmswVTWO257/TKeImkBBICJxK79dJr86ZzAfLd/O3a/pKqKjlRJ0ASIiTXFBQTeWbNjNY/9dxaDO2Zw/smvQJcUdHRGISNy748yBjOnVjp/9YyGfr9sVdDlxR0EgInEvNTmJv10ygg6t0vnB0/M0MF0jKQhEpFlom5XG1EsL2FFawc9eXKi5CxpBQSAizcagLtncflp/3lyymefnFQVdTtxQEIhIs3LFuJ6M7tWWX760mHU7yoIuJy4oCESkWUlKMu6bNJQkM3703GdU1+gU0ZEoCESk2enaJpNfnn0Mn6zeycMfrAi6nJinIBCRZunc4XmcMaQTf3hzGfPWaJTSw1EQiEizZGbcfd6xdMrJ4Kbp89lVVhF0STFLQSAizVZOi1T+dvEItpaUc9tzn+uS0kNQEIhIsza0W2vuOGMg7yzdwqMfrgq6nJikIBCRZu+ysT047ZhO3PP6UvUX1ENBICLNnplxz3ePpXPrDG6e8an6C+pQEIhIQshpkcpfLxrBlj37+MkLC9RfEEZBICIJY2i31vz09IG8tWQz0z5aHXQ5MUNBICIJ5YpxPThlYEfufu0LFhRpyGpQEIhIgjEz7pt0LB1aZXDD9Pns3lcZdEmBUxCISMJpnZnGny8azoZd+zRkNQoCEUlQI7u34bZT+/HKgo3MmLsu6HICpSAQkYR17Ym9OaFve3758mKWbtoddDmBURCISMJKSjL++L1hZLdI5cbpn1JWURV0SYFQEIhIQmvfMp37vzeMFVtL+L//XJyQ/QUKAhFJeOP6tOeWiX2ZOb+IZ+esDbqcqFMQiIgAN0/oy/j+ufzy5cXMX5tY4xEpCEREONBf0Ckng+ufmc/WPeVBlxQ1CgIREU/rzDQemjySnWUV3DRjPlXVNUGXFBUKAhGRMMd0yeHu84Ywe+UOfv5SYnQepwRdgIhIrDlvRFeWbS7hofdXkN82kx+c1DvoknylIBARqcft3+rPup1l3P3aUrq2yeTMYzsHXZJvFAQiIvVISjJ+P2kom4r3cetzn9EpJ52R3dsGXZYvfO0jMLNbzWyxmS0ysxlmlnGIdt81M2dmBX7WIyLSGBmpyTzy/QK65GRw1ZOFrN5WGnRJvvAtCMwsD7gZKHDODQaSgQvradfKazfHr1pERI5W26w0pl1+PA644olPmuU0l35fNZQCtDCzFCAT2FBPm18B9wL7fK5FROSo9GyfxdRLCyjauZcfPD2PiqrmdVmpb0HgnFsP3AesBTYCxc65N8PbmNlwoJtz7t+H25aZXWNmhWZWuHXrVr9KFhE5pON7tuXe7x7LnFU7+OmLzWvOYz9PDbUBzgZ6Al2ALDObHLY+CfgjcNuRtuWcm+qcK3DOFeTm5vpVsojIYZ0zPI9bT+nHi/PX8+d3lgddTsT4eWroFGCVc26rc64SeBEYG7a+FTAYeM/MVgOjgZfUYSwisezmiX04f0RX/vj2Mp4rbB4T2vh5+ehaYLSZZQJ7gYlAYe1K51wx0L72tZm9B/zYOVeIiEiMMjPuPm8IW/aEprnsmJ3BSf3i+0yFn30Ec4AXgPnAQm9fU83sLjM7y6/9ioj4LS0liQcuGUH/jq24/pl5LFpfHHRJTWLx1uFRUFDgCgt10CAiwdu8ex/nPfAxFdU1PPeDMfRsnxV0SYdkZvOcc/WeetegcyIiR6ljdgZPXnEc1TWOi6bOjtsbzhQEIiJN0KdDK569ahTlVdVc9Mhs1m4vC7qkRlMQiIg00cDO2Tx71Wj2VobCYN2O+AoDBYGISAQM6pLNM1eOoqS8igunxlcYKAhERCJkcF4Oz141itKKKr738CzWbI+PPgMFgYhIBNWGwd7Kai6Mkw5kBYGISIQd0yWH6VePpryqhu9NncWqGA8DBYGIiA8Gds5mxtWjqax2XBLjHcgKAhERn/Tv1IpnrhxFaUU1Fz86m43Fe4MuqV4KAhERHw3qks1TVxzPrtJKLnlkDlv2xN7UKwoCERGfDe3WmmmXH8em3fuY/OgctpWUB13SQRQEIiJRUNCjLY9eVsDaHWVcNHU2W/fEThgoCEREomRs7/Y8PuU41u0s46JHZsfMaSIFgYhIFI3t3Z5pU45n/c69XDR1Nlt2Bx8GCgIRkSgb07sdT1x+HBuL9/Hdh2axcmtJoPUoCEREAjCqVzuevSo0NtH5D37MvDU7AqtFQSAiEpDh+W148bqx5LRI5eJH5vDawo2B1KEgEBEJUI/2Wbx4/TiO6ZLN9dPn86e3v6KmJrozRyoIREQC1jYrjelXj+acYXn88e1lTHniE3aUVkRt/xw+rvcAAAivSURBVAoCEZEYkJGazB8uGMpvzh3C7BXb+fafP2T+2p1R2beCQEQkRpgZF4/KZ+Z1Y0lONiY9NIvfv/klFVU1vu5XQSAiEmOGdM3h3zedwLnD8/jLf5Zz1l//y6L1xb7tT0EgIhKDclqkct+koTw+pYAdpRWc/bePeOy/q3zZl4JARCSGTRjQkbduPYmzh3WhZ/tMX/aR4stWRUQkYnIyU/nDBcN8276OCEREEpyCQEQkwSkIREQSnIJARCTBKQhERBKcgkBEJMEpCEREEpyCQEQkwZlz0R33uqnMbCuwC6g78EbOEZYd6Xnt1/bAtqMorb79N2R93eWHe1231vBlR1N3NGsOfx7E91qfD30+Drc+Hj8fjakZoK9zLqferTvn4u4BTG3ssiM9D/taGKmaGrK+7vLDva5ba1PrjmbNQX+v9fnQ56O5fT4aU/OR9hGvp4ZePoplR3pe3/ubWlND1tddfrjX9dXalLqjWXP48yC+1/p8NJ4+Hw1/Hus1H3YfcXdqyG9mVuicKwi6jsaKx7pVc/TEY92qOXri9YjAT1ODLuAoxWPdqjl64rFu1RwlOiIQEUlwOiIQEUlwCgIRkQTXrIPAzB43sy1mtugo3jvSzBaa2XIz+7OZWdi6m8zsSzNbbGb3RrZqf+o2s1+Y2Xoz+8x7nBHrNYet/7GZOTNrH7mKffs+/8rMFnjf4zfNrEsc1Pw7M1vq1f0PM2sdyZp9rHuS9zNYY2YR66BtSq2H2N5lZvaV97gsbPlhP/dRdTTXvMbLAzgRGAEsOor3zgXGAAa8BpzuLR8PvA2ke687xEndvwB+HE/fa29dN+ANYA3QPtZrBrLD2twMPBQHNZ8KpHjP7wHuiYfPBzAQ6A+8BxQEXatXR486y9oCK72vbbznbQ737wri0ayPCJxzHwA7wpeZWW8ze93M5pnZh2Y2oO77zKwzoR/oWS70P/YUcI63+jrgt865cm8fW+Kkbl/5WPMfgduBiF/V4EfNzrndYU2zIl23TzW/6Zyr8prOBrpGsmYf6/7COfdlrNR6CN8C3nLO7XDO7QTeAk4L8me1Ps06CA5hKnCTc24k8GPggXra5AFFYa+LvGUA/YATzGyOmb1vZsf5Wu0BTa0b4Ebv8P9xM2vjX6n7NalmMzsLWO+c+9zvQsM0+ftsZr82s3XAJcCdPtZaKxKfjVpXEPrrNBoiWbffGlJrffKAdWGva+uPlX8XkGCT15tZS2As8HzY6bj0+prWs6z2L7sUQod4o4HjgOfMrJeX6r6IUN0PAr/yXv8K+D2hH3pfNLVmM8sE7iB02iIqIvR9xjl3B3CHmf0MuBH4eYRLPVBIhGr2tnUHUAU8G8ka6xPJuv12uFrN7HLgFm9ZH+BVM6sAVjnnzuXQ9Qf+7wqXUEFA6Ahol3NuWPhCM0sG5nkvXyL0SzP88LgrsMF7XgS86P3in2tmNYQGmtoay3U75zaHve8R4N8+1gtNr7k30BP43Pvh6wrMN7PjnXObYrTmuqYDr+BjEBChmr1OzG8DE/38oyZMpL/Xfqq3VgDn3DRgGoCZvQdMcc6tDmtSBJwc9rorob6EIoL/dx0QVOdEtB5AD8I6fYCPgUnecwOGHuJ9nxD6q7+2I+cMb/m1wF3e836EDvssDuruHNbmVuB/Y73mOm1WE+HOYp++z33D2twEvBAHNZ8GLAFyI11rND4fRLiz+Ghr5dCdxasInUVo4z1v29DPfbQegew0av84mAFsBCoJJfCVhP7KfB343Pvw33mI9xYAi4AVwF85cBd2GvCMt24+MCFO6n4aWAgsIPSXVudYr7lOm9VE/qohP77PM73lCwgN8pUXBzUvJ/QHzWfeI6JXOvlY97netsqBzcAbQdZKPUHgLb/C+x4vBy5vzOc+Wg8NMSEikuAS8aohEREJoyAQEUlwCgIRkQSnIBARSXAKAhGRBKcgkGbBzEqivL9HzWxQhLZVbaHRSheZ2ctHGv3TzFqb2fWR2LcIaIYyaSbMrMQ51zKC20txBwZi81V47Wb2JLDMOffrw7TvAfzbOTc4GvVJ86cjAmm2zCzXzGaa2SfeY5y3/Hgz+9jMPvW+9veWTzGz583sZeBNMzvZzN4zsxcsNF7/s7VjxnvLC7znJd5Ac5+b2Wwz6+gt7+29/sTM7mrgUcssDgy619LM3jGz+RYat/5sr81vgd7eUcTvvLY/8fazwMx+GcFvoyQABYE0Z38C/uicOw44H3jUW74UONE5N5zQ6KC/CXvPGOAy59wE7/Vw4IfAIKAXMK6e/WQBs51zQ4EPgKvD9v8nb/9HHEfGG2dnIqE7vwH2Aec650YQmgfj914Q/RRY4Zwb5pz7iZmdCvQFjgeGASPN7MQj7U+kVqINOieJ5RRgUNiIkdlm1grIAZ40s76ERnxMDXvPW8658LHo5zrnigDM7DNCY9D8t85+KjgwiN884Jve8zEcGGN+OnDfIepsEbbteYTGrIfQGDS/8X6p1xA6UuhYz/tP9R6feq9bEgqGDw6xP5GDKAikOUsCxjjn9oYvNLO/AO865871zre/F7a6tM42ysOeV1P/z0ylO9DZdqg2h7PXOTfMzHIIBcoNwJ8JzWeQC4x0zlWa2Wogo573G3C3c+7hRu5XBNCpIWne3iQ0HwAAZlY7jHAOsN57PsXH/c8mdEoK4MIjNXbOFROa3vLHZpZKqM4tXgiMB7p7TfcArcLe+gZwhTduPmaWZ2YdIvRvkASgIJDmItPMisIePyL0S7XA60BdQmgIcYB7gbvN7CMg2ceafgj8yMzmAp2B4iO9wTn3KaERLi8kNEFMgZkVEjo6WOq12Q585F1u+jvn3JuETj3NMrOFwAscHBQih6XLR0V84s2yttc558zsQuAi59zZR3qfSLSpj0DEPyOBv3pX+uzCx6lBRZpCRwQiIglOfQQiIglOQSAikuAUBCIiCU5BICKS4BQEIiIJ7v8Du19L6Orq54YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-03\n",
    "lr *= bs/48\n",
    "wd = 0.01 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='8' class='' max='10', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      80.00% [8/10 3:32:56<53:14]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>9.210296</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.543945</td>\n",
       "      <td>26:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>9.210299</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.534180</td>\n",
       "      <td>26:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>9.210279</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.534180</td>\n",
       "      <td>26:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>9.210284</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.534180</td>\n",
       "      <td>26:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>9.210316</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.534180</td>\n",
       "      <td>26:38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>9.210309</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.534180</td>\n",
       "      <td>26:38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>9.210290</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.534180</td>\n",
       "      <td>26:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>9.210315</td>\n",
       "      <td>9.210295</td>\n",
       "      <td>0.999284</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>9999.534180</td>\n",
       "      <td>26:38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='1826' class='' max='3801', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      48.04% [1826/3801 12:18<13:19 9.2103]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQ5UlEQVR4nO3df2yVhX7H8c/XUmnLDym1aqW61pvFH2ADtTLuMITIkgt4pyY22kWXu7slJHrvRLJlYm4y9Y8lzv1yZlOD93J/zEauVg3bok7dhbBNxBWttVAdoHipqBTuQFhAUb/74zzgaTltT8vz9Hwvfb+Sps95znOe5/M8nPPhnOfHqbm7AABxnVXqAACA4VHUABAcRQ0AwVHUABAcRQ0AwU3KYqZlVef4pHPOy2LWRanTAVXa5yVbPvDr4KuyCk07v6HUMZDYunXrfnevLXRfJkU959Jv6IUN/5XFrAGkpKK8TDOnnF3qGEiY2QdD3ZdJUZeXnaULZ1RmMWsAmHDYRw0AwVHUABBcJrs+AGA0jh8/rr6+Ph07dqzUUTJXUVGh+vp6lZeXF/0YihpAyfX19WnatGlqaGiQmZU6TmbcXQcOHFBfX58aGxuLfhy7PgCU3LFjx1RTU3NGl7QkmZlqampG/cmBogYQwple0ieMZT0pagAIjqIGMOEdPHhQjzzyyKgft3z5ch08eDCDRANR1AAmvKGK+ssvvxz2cc8//7xmzJiRVayTOOsDwIS3evVq7dq1S3PnzlV5ebmmTp2quro6dXV1afv27brxxhu1Z88eHTt2TCtXrtSKFSskSQ0NDers7NSRI0e0bNkyXXPNNXr11Vc1a9YsrV+/XpWV6VyhTVEDCOX+f9mm7Xs/TXWeV1w4Xff+7uwh73/ggQfU09Ojrq4ubdy4Udddd516enpOnkK3du1azZw5U0ePHtXVV1+tm266STU1NQPmsWPHDj355JN6/PHHdfPNN+uZZ57Rbbfdlkp+ihoABpk/f/6A85wffvhhPffcc5KkPXv2aMeOHacUdWNjo+bOnStJuuqqq7R79+7U8lDUAEIZ7p3veJkyZcrJ4Y0bN+qVV17R5s2bVVVVpcWLFxc8D3ry5Mknh8vKynT06NHU8nAwEcCEN23aNB0+fLjgfYcOHVJ1dbWqqqr0zjvv6LXXXhvndLyjBgDV1NRo4cKFmjNnjiorK3X++eefvG/p0qV67LHH1NTUpEsvvVQLFiwY93zm7qnPtKWlxTs7O1OfL4AzU29vry6//PJSxxg3hdbXzLa6e0uh6dn1AQDBUdQAEBxFDQDBUdQAEBxFDQDBUdQAEBxFDQBjMHXqVEnS3r171draWnCaxYsXK41TlYsqajNbZWbbzKzHzJ40s4rTXjIAnAEuvPBCdXR0ZLqMEYvazGZJulNSi7vPkVQmqS3TVAAwzu6+++4B30l933336f7779eSJUvU3NysK6+8UuvXrz/lcbt379acOXMkSUePHlVbW5uampp0yy23pPZ9H8VeQj5JUqWZHZdUJWlvKksHgMFeWC19/Ha687zgSmnZA8NO0tbWprvuukt33HGHJOmpp57Siy++qFWrVmn69Onav3+/FixYoOuvv37Iv3v46KOPqqqqSt3d3eru7lZzc3Mq8Ucsanf/0Mz+WtIvJR2V9JK7vzR4OjNbIWmFJF188cWphAOA8TJv3jzt27dPe/fuVX9/v6qrq1VXV6dVq1Zp06ZNOuuss/Thhx/qk08+0QUXXFBwHps2bdKdd94pSWpqalJTU1Mq2UYsajOrlnSDpEZJByU9bWa3ufsT+dO5+xpJa6Tcd32kkg7AxDPCO98stba2qqOjQx9//LHa2trU3t6u/v5+bd26VeXl5WpoaCj4Faf5svhr6sUcTPwdSe+7e7+7H5f0rKTfTj0JAJRYW1ub1q1bp46ODrW2turQoUM677zzVF5erg0bNuiDDz4Y9vGLFi1Se3u7JKmnp0fd3d2p5CqmqH8paYGZVVnuv4olknpTWToABDJ79mwdPnxYs2bNUl1dnW699VZ1dnaqpaVF7e3tuuyyy4Z9/O23364jR46oqalJDz74oObPn59KrmL2UW8xsw5Jb0j6QtKbSnZxAMCZ5u23vz6Qee6552rz5s0Fpzty5Iik3B+47enpkSRVVlZq3bp1qWcq6qwPd79X0r2pLx0AMCKuTASA4ChqACFk8demIhrLelLUAEquoqJCBw4cOOPL2t114MABVVSM7ls4+OO2AEquvr5efX196u/vL3WUzFVUVKi+vn5Uj6GoAZRceXm5GhsbSx0jLHZ9AEBwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwRRW1mc0wsw4ze8fMes3sm1kHAwDkTCpyur+X9KK7t5rZ2ZKqMswEAMgzYlGb2XRJiyT9gSS5++eSPs82FgDghGJ2fVwiqV/Sj83sTTP7oZlNGTyRma0ws04z6+zv7089KABMVMUU9SRJzZIedfd5kv5P0urBE7n7GndvcfeW2tralGMCwMRVTFH3Sepz9y3J7Q7lihsAMA5GLGp3/1jSHjO7NBm1RNL2TFMBAE4q9qyPP5bUnpzx8Z6k72YXCQCQr6iidvcuSS0ZZwEAFMCViQAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMFR1AAQHEUNAMEVXdRmVmZmb5rZv2YZCAAw0GjeUa+U1JtVEABAYUUVtZnVS7pO0g+zjQMAGKzYd9QPSfozSV8NNYGZrTCzTjPr7O/vTyUcAKCIojazb0va5+5bh5vO3de4e4u7t9TW1qYWEAAmumLeUS+UdL2Z7Za0TtK1ZvZEpqkAACeNWNTufo+717t7g6Q2Sb9w99syTwYAkMR51AAQ3qTRTOzuGyVtzCQJAKAg3lEDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAER1EDQHAUNQAEN2JRm9lFZrbBzHrNbJuZrRyPYACAnElFTPOFpD9x9zfMbJqkrWb2srtvzzgbAEBFvKN294/c/Y1k+LCkXkmzsg4GAMgZ1T5qM2uQNE/SlizCAABOVXRRm9lUSc9IusvdPy1w/woz6zSzzv7+/jQzAsCEVlRRm1m5ciXd7u7PFprG3de4e4u7t9TW1qaZEQAmtGLO+jBJP5LU6+5/m30kAEC+Yt5RL5T0+5KuNbOu5Gd5xrkAAIkRT89z9/+UZOOQBQBQAFcmAkBwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwFDUABEdRA0BwRRW1mS01s3fNbKeZrc46FADgayMWtZmVSfpHScskXSHp98zsiqyDAQByinlHPV/STnd/z90/l7RO0g3ZxgIAnDCpiGlmSdqTd7tP0m8NnsjMVkhakdz8zMx6Tj/euDlX0v5ShxgF8maLvNkib2G/MdQdxRS1FRjnp4xwXyNpjSSZWae7txQdr8TImy3yZou82YqQt5hdH32SLsq7XS9pbzZxAACDFVPU/y3pN82s0czOltQm6Z+zjQUAOGHEXR/u/oWZfV/Sv0kqk7TW3beN8LA1aYQbR+TNFnmzRd5slTyvuZ+yuxkAEAhXJgJAcBQ1AASXalFHutTczHab2dtm1mVmncm4mWb2spntSH5XJ+PNzB5OcnebWXPefL6TTL/DzL6TYr61ZrYv/3zzNPOZ2VXJ+u9MHlvoNMvTzXufmX2YbOMuM1ued989ybLfNbNv5Y0v+BxJDlZvSdbj58mB69PJe5GZbTCzXjPbZmYrk/Eht/EweUNuYzOrMLPXzeytJO/9wy3DzCYnt3cm9zeMdT1SzvsTM3s/b/vOTcaX/DU3gLun8qPcgcZdki6RdLaktyRdkdb8x5Bnt6RzB417UNLqZHi1pL9MhpdLekG5c8YXSNqSjJ8p6b3kd3UyXJ1SvkWSmiX1ZJFP0uuSvpk85gVJyzLIe5+kPy0w7RXJv/9kSY3J86JsuOeIpKcktSXDj0m6/TTz1klqToanSfqfJFfIbTxM3pDbOFnnqclwuaQtyXYruAxJd0h6LBluk/Tzsa5Hynl/Iqm1wPQlf83l/6T5jvrX4VLzGyT9NBn+qaQb88b/zHNekzTDzOokfUvSy+7+K3f/X0kvS1qaRhB33yTpV1nkS+6b7u6bPfcM+lnevNLMO5QbJK1z98/c/X1JO5V7fhR8jiTvPK6V1FFg3cea9yN3fyMZPiypV7mrbENu42HyDqWk2zjZTkeSm+XJjw+zjPzt3iFpSZJpVOuRQd6hlPw1ly/Noi50qflwT7SsuaSXzGyr5S5vl6Tz3f0jKffCkHReMn6o7OO9Tmnlm5UMDx6fhe8nHw3XntiNMIa8NZIOuvsXWeRNPmbPU+5dVPhtPCivFHQbm1mZmXVJ2qdcYe0aZhkncyX3H0oyjdtrb3Bedz+xff8i2b5/Z2aTB+ctMlemr7k0i7qoS83H0UJ3b1buW/++Z2aLhpl2qOxR1mm0+cYr96OSviFprqSPJP1NMj5MXjObKukZSXe5+6fDTTrKbJlkLpA37DZ29y/dfa5yVyvPl3T5MMsIl9fM5ki6R9Jlkq5WbnfG3VHy5kuzqENdau7ue5Pf+yQ9p9wT6ZPkI4qS3/uSyYfKPt7rlFa+vmR48PhUufsnyZP/K0mPK7eNx5J3v3IfLScNGn9azKxcudJrd/dnk9Fht3GhvNG3cZLxoKSNyu3LHWoZJ3Ml95+j3K60cX/t5eVdmuxycnf/TNKPNfbtm+1rLq2d3cpd5fiecgcETuz8n53W/EeZZYqkaXnDryq3b/mvNPBA0oPJ8HUaeODgdf/6wMH7yh00qE6GZ6aYs0EDD86llk+5S/8X6OsDG8szyFuXN7xKuX2NkjRbAw8QvafcwaEhnyOSntbAg1B3nGZWU24/4UODxofcxsPkDbmNJdVKmpEMV0r6D0nfHmoZkr6ngQcTnxrreqScty5v+z8k6YEIz4dT8qc1oyTocuWOVu+S9IM05z3KHJck/7BvSdp2Ioty+8T+XdKO5PeJDWzK/XGEXZLeltSSN68/VO4Ax05J300x45PKfZQ9rtz/xn+UZj5JLZJ6ksf8g5KrUFPO+09Jnm7lvv8lv1R+kCz7XeUd/R7qOZL8m72erMfTkiafZt5rlPvo2S2pK/lZHnUbD5M35DaW1CTpzSRXj6Q/H24ZkiqS2zuT+y8Z63qknPcXyfbtkfSEvj4zpOSvufwfLiEHgOC4MhEAgqOoASA4ihoAgqOoASA4ihoAgqOoASA4ihoAgvt/2gXPgELC7VoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.0007163433474488556.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "learn.fit_one_cycle(10, lr, wd=wd, moms=(0.8,0.7), \n",
    "                    callbacks=[ShowGraph(learn),\n",
    "                               SaveModelCallback(learn.to_fp32(),\n",
    "                                                 monitor='accuracy',\n",
    "                                                 name='bestmodel_sp15_multifit')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mdl_path = path/'models'\n",
    "mdl_path.mkdir(exist_ok=True)\n",
    "learn.to_fp32().save(mdl_path/lm_fns3[0], with_opt=False)\n",
    "learn.data.vocab.save(mdl_path/(lm_fns3[1] + '.pkl'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:root] *",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}