coronaindestad.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CoronaInDeStad.nl Web Crawler\n",
    "\n",
    "Web crawler for the website coronaindestad.nl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Retrieve html documents from website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"CoronaInDeStad.nl Web Crawler\"\"\"\n",
    "\n",
    "import datetime\n",
    "import os\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as mdates\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# De collectie\n",
    "# URL = \"https://www.coronaindestad.nl/small-poetic-sonic-events-2/\"\n",
    "# De tentoonstelling\n",
    "# URL = \"https://www.coronaindestad.nl/meervaart-brief-van-floris-van-delft/\"\n",
    "# Veerkrachtige verhalen uit Amsterdam-West\n",
    "# URL = \"https://www.coronaindestad.nl/corona-getuigenissen-verhaal-yurdakul/?h=3364\"\n",
    "# Over de tentoonstellin / de gehele collectie\n",
    "# URL = \"https://www.coronaindestad.nl/van-corona-naar-familie-k/\"\n",
    "# texts only\n",
    "# URL = \"https://www.coronaindestad.nl/onderweg-naar-de-datsja/\"\n",
    "# De tentoonstelling\n",
    "# URL = \"https://www.coronaindestad.nl/knuffelen/\"\n",
    "# Corona in de stad\n",
    "# URL = \"https://www.coronaindestad.nl/sound-of-silence-chin-chin-club/\"\n",
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/rvdk-ronald-van-der-kemp-army-of-love-surrender-to-a-sustainble-future/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Zaaloverzicht\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/corona-en-de-jeugd/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/the-world-after-3-the-world-after/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/frigiti-tori-untold-stories-shared-history-muriel-3/\"\n",
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/schrijven-naar-de-toekomst-laurens-en-petra-de-boer-amsterdam-en-ijmuiden/\")\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/psychiatire-in-de-stad-een-nieuwsbrief/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/freedom-4/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/corona-streetart-ndsm/?h=1573\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/stroopwafels/?h=3675\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/where-is-my-nature-join-us-on-the-3rd-of-july-teaser/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/briefwisseling-met-onze-overburen/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/was-het-echt-een-jaar/?h=969\"\n",
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/steunpilaren-van-de-stad-aby-patat-in-nieuw-west/?h=4249\")\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/mannen-in-nieuw-west-diwan/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/avondklok-23-24-januari-2021-dam-amsterdam/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/linnaeusparkweg-oost/?h=3710\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/wemakethe-city-reset/?h=1552\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/houd-1-5-meter-afstand-tas/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/gluklya-corona-diary-2021/?h=3712\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/the-clarinet-player/?h=4027\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/koosjere-vrijheidsmaaltijdsoep/?h=1908\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/joods-verzetsmonument/?h=1908\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/verklaring-minjan-nihs-en-pig/?h=1908\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/hoegaathetechtmetje-59/?h=3113\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/blijf-staan/?h=2449\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/toegankelijkheid-stadsloketten-vergroot/?h=1603\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/1-uitdelen-laptops-en-thuiswerkboeken/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/noraly-beyer-love-after-love/\"\n",
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/6-vuilnis-7-tussen-kunst-en-quarantaine-8-oesters-en-champagne/?h=1089\")\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/protesten-op-het-museumplein-21-februari-2021/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/koningsdagklokgelui/\"\n",
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/aflevering-6-cody-hostenbach-over-ongelijkheid-in-de-stad/?h=3784\")\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/22217-2/?h=3871\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/we-mogen-weer/?h=3804\"\n",
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/straks-eindelijk-weer-samen-muziek-maken-voor-met-en-door-het-publiek/?h=3804\")\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/meervaart-lockdown-battle/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/opkikkeractie-in-de-aker/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/de-middag-na-de-dag-ervoor/?h=2697\"\n",
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/black-pride-nl-protest-against-anti-black-queer-trans-violence-4/?h=3053\")\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/geur/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/on-our-way-to-the-client/?h=3736\"\n",
    "# most recent\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/corona-in-de-stad-4/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/no-one-knows/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/coronacrisiskruk/\"\n",
    "INITIAL_URL = \"https://www.coronaindestad.nl/de-kunst-van-het-delen-in-tijden-van-corona/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "INITIAL_URL = (\"https://www.coronaindestad.nl\" +\n",
    "               \"/horse-with-no-name-how-not-to-make-money-with-a-viral-video/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_next_link(web_page, target_class=\"next-button\"):\n",
    "    \"\"\" get next link from html document\"\"\"\n",
    "    page_links = []\n",
    "    for tag_a in BeautifulSoup(web_page, \"html.parser\").select('a'):\n",
    "        a_class = tag_a.get(\"class\")\n",
    "        if isinstance(a_class, list) and target_class in a_class:\n",
    "            page_links.append(tag_a.get(\"href\"))\n",
    "    return page_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_text(web_page, target_class=\"large-7\"):\n",
    "    \"\"\" extract text from web page \"\"\"\n",
    "    text = []\n",
    "    for tag_div in BeautifulSoup(web_page, \"html.parser\").select('div'):\n",
    "        div_class = tag_div.get(\"class\")\n",
    "        if isinstance(div_class, list) and target_class in div_class:\n",
    "            text.append(tag_div.text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = \"../coronaindestad.nl/scraped/\"\n",
    "\n",
    "def store_web_page(web_page, file_name_):\n",
    "    \"\"\" store webpage on disk \"\"\"\n",
    "    out_file = open(file_name_, \"w\")\n",
    "    print(web_page.content, file=out_file)\n",
    "    out_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_web_page(url):\n",
    "    \"\"\" fetch webpage from website \"\"\"\n",
    "    time.sleep(1)\n",
    "    start_time = time.time()\n",
    "    web_page = requests.get(url)\n",
    "    fetch_time = int(time.time() - start_time)\n",
    "    print(f\"{fetch_time} {url}\")\n",
    "    web_page_date = get_web_page_date(web_page.content)\n",
    "    return web_page, web_page_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_out_file_name(url):\n",
    "    \"\"\" convert url to file name \"\"\"\n",
    "    dir_name = url.split('/')[3]\n",
    "    if dir_name == \"en\":\n",
    "        dir_name = url.split('/')[4]\n",
    "    return f\"{DATA_DIR}{dir_name}.html\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_file(file_name_):\n",
    "    \"\"\" read file from disk \"\"\"\n",
    "    in_file = open(file_name_, \"r\")\n",
    "    file_text_ = in_file.read()\n",
    "    in_file.close()\n",
    "    file_date = get_web_page_date(file_text_)\n",
    "    return file_text_, file_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_web_page_date(web_page):\n",
    "    \"\"\" extract date from htmlpage \"\"\"\n",
    "    page_date = \"\"\n",
    "    try:\n",
    "        for page_meta in BeautifulSoup(web_page, \"html.parser\").select('meta'):\n",
    "            if (\"property\" in page_meta.attrs and\n",
    "                    page_meta.attrs[\"property\"] == \"article:published_time\"):\n",
    "                page_date = page_meta.attrs[\"content\"]\n",
    "    except Exception as _:\n",
    "        pass\n",
    "    return page_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_web_page_tree(url, initial_url=INITIAL_URL):\n",
    "    \"\"\" retrieve linked webpages from website\"\"\"\n",
    "    file_name_ = make_out_file_name(url)\n",
    "    if not os.path.isfile(file_name_) or url == initial_url:\n",
    "        if os.path.isfile(file_name_):\n",
    "            web_page_text, _ = read_file(file_name_)\n",
    "        else:\n",
    "            web_page, _ = fetch_web_page(url)\n",
    "            store_web_page(web_page, file_name_)\n",
    "            web_page_text = web_page.content\n",
    "        try:\n",
    "            next_url_ = get_next_link(web_page_text, target_class=\"next-button\")[0]\n",
    "            get_web_page_tree(next_url_, initial_url)\n",
    "        except Exception as _:\n",
    "            pass\n",
    "        try:\n",
    "            prev_url_ = get_next_link(web_page_text, target_class=\"back-button\")[0]\n",
    "            get_web_page_tree(prev_url_, initial_url)\n",
    "        except Exception as _:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_web_page_tree(INITIAL_URL, initial_url=INITIAL_URL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Double check navigation links in retrieved documents\n",
    "\n",
    "It is possible to disable specific lint warnings like this: `# pylint: disable=C0103`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "file_names = os.listdir(DATA_DIR)\n",
    "file_counter = 0\n",
    "for file_name in file_names:\n",
    "    file_text, date = read_file(DATA_DIR + file_name)\n",
    "    try:\n",
    "        next_url = get_next_link(file_text, target_class=\"next-button\")[0]\n",
    "        get_web_page_tree(next_url, initial_url=next_url)\n",
    "    except Exception as _:\n",
    "        pass\n",
    "    try:\n",
    "        prev_url = get_next_link(file_text, target_class=\"back-button\")[0]\n",
    "        get_web_page_tree(prev_url, initial_url=prev_url)\n",
    "    except Exception as _:\n",
    "        pass\n",
    "    file_counter += 1\n",
    "    if file_counter % 100 == 0:\n",
    "        print(file_counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1688"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_names = os.listdir(DATA_DIR)\n",
    "len(file_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Summarize document data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squeal(text=None):\n",
    "    \"\"\" show progress of process \"\"\"\n",
    "    clear_output(wait=True)\n",
    "    if not text is None:\n",
    "        print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_months(file_names_):\n",
    "    \"\"\" extract dates from files stored on disk \"\"\"\n",
    "    file_months = {}\n",
    "    counter = 0\n",
    "    for file_name_ in file_names_:\n",
    "        _, file_date = read_file(DATA_DIR + file_name_)\n",
    "        year = file_date[0:4]\n",
    "        month = file_date[5:7]\n",
    "        file_date = year + month\n",
    "        if file_date not in file_months:\n",
    "            file_months[file_date] = 1\n",
    "        else:\n",
    "            file_months[file_date] += 1\n",
    "        counter += 1\n",
    "        if counter % 100 == 0:\n",
    "            squeal(counter)\n",
    "    squeal(counter)\n",
    "    return file_months"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_months(file_months, lockdown_starts):\n",
    "    \"\"\" visualize frequencies of documents per month \"\"\"\n",
    "    data_x = [datetime.datetime.strptime(key + \"16\", \"%Y%m%d\")\n",
    "              for key in sorted(file_months.keys())]\n",
    "    data_y = [file_months[key] for key in sorted(months.keys())]\n",
    "    plt.figure(figsize=(6, 3))\n",
    "    plot_ax = plt.subplot(111)\n",
    "    plot_ax.bar(data_x, data_y, width=31)\n",
    "    plot_ax.scatter([datetime.datetime.strptime(d, \"%Y%m%d\") for d in lockdown_starts],\n",
    "                    [0 for d in lockdown_starts],\n",
    "                    label=\"lockdown starts\")\n",
    "    plot_ax.xaxis_date()\n",
    "    plot_ax.legend()\n",
    "    plot_ax.xaxis.set_major_formatter(mdates.DateFormatter(\"%-m/%-y\"))\n",
    "    plt.xlabel(\"months\")\n",
    "    plt.title(\"Number of documents per month\")\n",
    "    plt.savefig(\"coronaindestad.png\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_types(file_names_):\n",
    "    \"\"\" extract document types from files stored on disk \"\"\"\n",
    "    file_types_ = {}\n",
    "    counter = 0\n",
    "    for file_name_ in file_names_:\n",
    "        file_text_, _ = read_file(DATA_DIR + file_name_)\n",
    "        file_type = \"\"\n",
    "        for div in BeautifulSoup(file_text_, \"html.parser\").select('div'):\n",
    "            if \"class\" in div.attrs and \"tags\" in div.attrs[\"class\"]:\n",
    "                for span in BeautifulSoup(str(div), \"html.parser\").select('span'):\n",
    "                    file_type = span.text\n",
    "                    break\n",
    "                if file_type != \"\":\n",
    "                    if file_type in file_types_:\n",
    "                        file_types_[file_type] += 1\n",
    "                    else:\n",
    "                        file_types_[file_type] = 1\n",
    "                    break\n",
    "        if file_type == \"\":\n",
    "            print(f\"no file type found for file {file_name_}\")\n",
    "        counter += 1\n",
    "        if counter % 100 == 0:\n",
    "            squeal(counter)\n",
    "    squeal(counter)\n",
    "    return file_types_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1688\n"
     ]
    }
   ],
   "source": [
    "months = get_months(file_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAADgCAYAAADrA2ZFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAeF0lEQVR4nO3de5xVZd338c9XQPCUqCAqUGiZZYBoA6gpoaaPoqndlmlkQAgdNPOxR8PyZWbeBeaTRt1PZR7QMpG4S0m9OymEmCcwxHOgUgwiAgpBSnH4PX+sa3AxDjN79uw9B9b3/Xrt16x1reta67evfc1v1lxr77UVEZiZWbHs0NYBmJlZ63PyNzMrICd/M7MCcvI3MysgJ38zswJy8jczKyAnf0PSFElXtdGxJelmSa9LerSE+v0khaTOrRGftQ1JoyXNaes4tmdO/u2QpMWSXpW0S67sXEmz2jCsajkKOB7oExFD2jqY1taWf3jbC/9BbxtO/u1XJ+DLbR1Ec0nq1Mwm7wIWR8Q/qxGPlaaM1806OCf/9uu7wP+R1L3+hobOlCTNknRuWh4t6UFJ10paLelFSUem8iXpv4pR9XbbQ9IfJK2V9CdJ78rt+31p22uSnpd0Zm7bFEk/knSvpH8CxzQQ736SZqT2iySNS+VjgRuAIyStk/TNBtp2knSNpJWSXgROLmXfubZfk/RCel7zJPWtdP9J6ppi/Luk5ZJ+LGmntG24pFpJX0ntlkkak7aNB0YCl6Tn/5tU/lVJS1PMz0s6rn6/5Pr+x1V83WZJukrSn+vik7SXpNsk/UPSY5L65eofmcrWpJ9H1tvXt1K/rpX0e0k90ubZ6efqdJwjcu2uUTYl+JKkkxrqBytTRPjRzh7AYuAjwK+Aq1LZucCstNwPCKBzrs0s4Ny0PBrYCIwh+w/iKuDvwH8BXYETgLXArqn+lLQ+LG3/PjAnbdsFWJL21Rk4FFgJHJxruwb4ENnJRLcGns9s4P8B3YBBwArg2Fyscxrpi88DzwF9gT2Bmfnn3sS+LwaeBA4CBBwC7FWF/rsWmJHi2w34DfCdtG142teVQBdgBPAGsEeu/67KxXFQ6u/9cq/1u7fRN9V+3WYBi4B3A7sDzwB/JRubnYFbgZtT3T2B14Fz0raz0/peuX29ALwX2CmtT2xkPI8GNgDj0mvwBeBlQG39+7m9PNo8AD8aeFHeSv790y9oT5qf/Bfmtg1I9XvlylYBg9LyFGBqbtuuwCayhPtJ4IF68f0E+Eau7a2NPJe+aV+75cq+A0zJxdpY8r8f+Hxu/YS6517Cvp8HTmtgnxXrP7I/Kv8kl6CBI4CX0vJw4M16x3oVODzXf/nk/560/SNAlybGSdVet1yffD23/n+B/8mtfxSYn5bPAR6t1/4hYHRuX5fltn0R+G0jr8doYFFufedUZ5+2/v3cXh6+wNKORcRTku4GJgDPNrP58tzym2l/9ct2za0vyR13naTXgP3I5uSHSlqdq9sZ+FlDbRuwH/BaRKzNlf0NqCnhOdS1z+//b83Yd1+ys81ylNp/PckS0zxJddtEdrZaZ1VEbMytv8HWfb9FRCySdCFwBfABSb8DLoqIl7cRZ7Vetzr1n/O2xtB+bP3akNZ759ZfyS1vsw8aqh8Rb6T+baqNlchz/u3fN8j+9c3/EtVdHN05V7ZPC4/Tt25B0q5k/8a/TJYg/hQR3XOPXSPiC7m2jd0a9mVgT0m75creCSwtMa5l+dhS21L3vYRsyqK+SvbfSrIk+IFc/+weEaUmqbf1XUT8IiKOIkvgAUxqpH21XrfmejnFm1fq6+xbC7cBJ/92LiIWAXcAF+TKVpD9Un06XdT8LA0nueYYIekoSTsC3wIejoglwN3AeyWdI6lLegyW9P4S418C/Bn4jqRukgYCY4GflxjXNOACSX0k7UH2X1Cp+74B+JakA5UZKGmvSvZfRGwGfgpcK2lvAEm9Jf2vEnexHDigbkXSQZKOldQVWE/2h2VzI+2r8rqV4d50vE9J6izpk8DBKY6mrCB7jgc0VdEqx8m/Y7iS7AJe3jiyC5qrgA+QJcGW+AXZfxmvAR8EPg2QplROAM4iO7t7hexMtGsz9n022bzuy8Cvyead/1hi258CvwOeAB4nuwhe6r6/R/bH4/fAP4AbyS42QmX776tkF0YflvQP4I9kF25LcSNwcHpX0Z1k/TqR7D+KV4C9gUsbaV/N161kEbEKOAX4ClmfXgKcEhErS2j7BvCfwIOpHw6vRoy2NaWLKWbWwUiaAtRGxGVtHYt1PD7zNzMrICd/M7MC8rSPmVkB+czfzKyAnPzNzAqoXXzCt0ePHtGvX7+2DsPMrEOZN2/eyojoWU7bdpH8+/Xrx9y5c9s6DDOzDkVS/VtqlMzTPmZmBeTkb2ZWQE7+ZmYF1C7m/M2sOjZs2EBtbS3r169v61CsBbp160afPn3o0qVLxfbp5G+2HautrWW33XajX79+5L5vwDqQiGDVqlXU1tay//77V2y/Tv719JtwT7PbLJ54ctOVzNrA+vXrnfg7OEnstdderFixoqL79Zy/2XbOib/jq8Zr6ORvZlW1667lffPi8OHDm/z8z5QpUzj//PPL2n8lTJkyhZdf3tY3bG7bnXfeyTPPPFOFiErn5G9mVqZykv/GjRud/M2snVkwDa7tD1d0z34umFaxXUcEF198Mf3792fAgAHccccdW7ZNmjSJAQMGcMghhzBhwoSt2m3evJnRo0dz2WXZd9bcfPPNvPe972XIkCE8+OCDW+otXryYY489loEDB3Lcccfx97//nU2bNrH//vsTEaxevZpOnToxe/ZsAIYNG8bChQu54oor+OxnP8vw4cM54IADmDx58tti37RpE6NHj94S+7XXXsv06dOZO3cuI0eOZNCgQbz55ptceeWVDB48mP79+zN+/Hjq7po8fPhwLrzwQmpqapg0aRIzZszg4osvZtCgQbzwwgtMnjyZgw8+mIEDB3LWWWdVrM8b4wu+ZpZZMA1+cwFseDNbX7MkWwcYeGaLd/+rX/2K+fPn88QTT7By5UoGDx7MsGHDmD9/PnfddRePPPIIO++8M6+99tqWNhs3bmTkyJH079+fr3/96yxbtoxvfOMbzJs3j913351jjjmGQw89FIAvfelLjBo1ilGjRnHTTTdxwQUXcOedd3LQQQfxzDPP8NJLL3HYYYfxwAMPMHToUJYsWcKBBx4IwHPPPcfMmTNZu3YtBx10EF/4whe2elvl/PnzWbp0KU899RQAq1evpnv37vzwhz/kmmuuoaamBoDzzz+fyy+/HIBzzjmHu+++m49+9KMA/Pvf/94yjbVw4UJOOeUUPv7xjwMwceJEXnrpJbp27crq1atb3Nel8Jm/mWXuu/KtxF9nw5tZeQXMmTOHs88+m06dOtGrVy8+/OEP89hjj/HHP/6RMWPGsPPOOwOw5557bmnzuc99bkviB3jkkUcYPnw4PXv2ZMcdd+STn/zklroPPfQQn/rUp4As8c6ZMweAo48+mtmzZzN79mwuvfRS5syZw2OPPcbgwYO3tD355JPp2rUrPXr0YO+992b58uVbxX7AAQfw4osv8qUvfYnf/va3vOMd72jwOc6cOZOhQ4cyYMAA7r//fp5++ukt2/Kx1jdw4EBGjhzJz3/+czp3bp1zcid/M8usqW1eeSs48sgjmTlzZos+pDZs2DAeeOABHn30UUaMGMHq1auZNWsWRx999JY6Xbu+9b32nTp1YuPGjVvtY4899uCJJ55g+PDh/PjHP+bcc89923HWr1/PF7/4RaZPn86TTz7JuHHjtop7l1122WaM99xzD+eddx6PP/44gwcPftvxq8HJ38wyu/dpXnkzHX300dxxxx1s2rSJFStWMHv2bIYMGcLxxx/PzTffzBtvvAGw1bTP2LFjGTFiBGeeeSYbN25k6NCh/OlPf2LVqlVs2LCBX/7yl1vqHnnkkUydOhWA2267bUtyHzJkCH/+85/ZYYcd6NatG4MGDeInP/kJw4YNKzn2lStXsnnzZs444wyuuuoqHn/8cQB222031q5dC7Al0ffo0YN169Yxffr0be4v327z5s0sWbKEY445hkmTJrFmzRrWrVtXcmzl8py/mWWOu3zrOX+ALjtl5RXwsY99jIceeohDDjkESVx99dXss88+nHjiicyfP5+amhp23HFHRowYwbe//e0t7S666CLWrFnDOeecw2233cYVV1zBEUccQffu3Rk0aNCWej/4wQ8YM2YM3/3ud+nZsyc333wzkJ3V9+3bl8MPPxzI/gjdfvvtDBgwoOTYly5dypgxY9i8eTMA3/nOdwAYPXo0n//859lpp5146KGHGDduHP3792efffbZalqpvrPOOotx48YxefJkpk6dytixY1mzZg0RwQUXXED37t1Ljq1cJX2Hr6TFwFpgE7AxImok7QncAfQDFgNnRsTryj6N8H1gBPAGMDoiHm9s/zU1NdFe7ufvT/ja9uTZZ5/l/e9/f+kNFkzL5vjX1GZn/MddXpGLvdZyDb2WkuZFRE05+2vOmf8xEbEytz4BuC8iJkqakNa/CpwEHJgeQ4EfpZ9m1t4NPNPJviBaMud/GnBLWr4FOD1XfmtkHga6S9q3BccxM7MKKzX5B/B7SfMkjU9lvSJiWVp+BeiVlnsDS3Jta1OZmZm1E6VO+xwVEUsl7Q38QdJz+Y0REZKavniQk/6IjAd45zvf2ZymZtYMEeGbu3VwpVybba6SzvwjYmn6+Srwa2AIsLxuOif9fDVVXwr0zTXvk8rq7/P6iKiJiJqePcv68nkza0K3bt1YtWpVVZKHtY66+/l369atovtt8sxf0i7ADhGxNi2fAFwJzABGARPTz7tSkxnA+ZKmkl3oXZObHjKzVtSnTx9qa2srfi94a1113+RVSaVM+/QCfp3+bewM/CIifivpMWCapLHA34C6twjcS/Y2z0Vkb/UcU9GIzaxkXbp0qei3P9n2o8nkHxEvAoc0UL4KOK6B8gDOq0h0ZmZWFb69g5lZATn5m5kVkJO/mVkBOfmbmRWQk7+ZWQE5+ZuZFZCTv5lZATn5m5kVkJO/mVkBOfmbmRWQk7+ZWQE5+ZuZFZCTv5lZATn5m5kVkJO/mVkBOfmbmRWQk7+ZWQE5+ZuZFZCTv5lZAZWc/CV1kvQXSXen9f0lPSJpkaQ7JO2Yyrum9UVpe78qxW5mZmVqzpn/l4Fnc+uTgGsj4j3A68DYVD4WeD2VX5vqmZlZO1JS8pfUBzgZuCGtCzgWmJ6q3AKcnpZPS+uk7cel+mZm1k6UeuZ/HXAJsDmt7wWsjoiNab0W6J2WewNLANL2Nam+mZm1E00mf0mnAK9GxLxKHljSeElzJc1dsWJFJXdtZmZNKOXM/0PAqZIWA1PJpnu+D3SX1DnV6QMsTctLgb4AafvuwKr6O42I6yOiJiJqevbs2aInYWZmzdNk8o+ISyOiT0T0A84C7o+IkcBM4OOp2ijgrrQ8I62Ttt8fEVHRqM3MrEVa8j7/rwIXSVpENqd/Yyq/EdgrlV8ETGhZiGZmVmmdm67yloiYBcxKyy8CQxqosx74RAViMzOzKvEnfM3MCsjJ38ysgJz8zcwKyMnfzKyAnPzNzArIyd/MrICc/M3MCsjJ38ysgJz8zcwKyMnfzKyAnPzNzArIyd/MrICc/M3MCsjJ38ysgJz8zcwKyMnfzKyAnPzNzArIyd/MrICc/M3MCsjJ38ysgJpM/pK6SXpU0hOSnpb0zVS+v6RHJC2SdIekHVN517S+KG3vV+XnYGZmzVTKmf+/gGMj4hBgEHCipMOBScC1EfEe4HVgbKo/Fng9lV+b6pmZWTvSZPKPzLq02iU9AjgWmJ7KbwFOT8unpXXS9uMkqVIBm5lZy5U05y+pk6T5wKvAH4AXgNURsTFVqQV6p+XewBKAtH0NsFcD+xwvaa6kuStWrGjRkzAzs+YpKflHxKaIGAT0AYYA72vpgSPi+oioiYianj17tnR3ZmbWDM16t09ErAZmAkcA3SV1Tpv6AEvT8lKgL0DavjuwqhLBmplZZZTybp+ekrqn5Z2A44Fnyf4IfDxVGwXclZZnpHXS9vsjIioYs5mZtVDnpquwL3CLpE5kfyymRcTdkp4Bpkq6CvgLcGOqfyPwM0mLgNeAs6oQt5mZtUCTyT8iFgCHNlD+Itn8f/3y9cAnKhKdmZlVhT/ha2ZWQE7+ZmYF5ORvZlZATv5mZgXk5G9mVkBO/mZmBeTkb2ZWQE7+ZmYF5ORvZlZATv5mZgXk5G9mVkBO/mZmBeTkb2ZWQKXc0tma0G/CPc1us3jiyVWIxMysND7zNzMrICd/M7MCcvI3MysgJ38zswJy8jczK6Amk7+kvpJmSnpG0tOSvpzK95T0B0kL0889UrkkTZa0SNICSYdV+0mYmVnzlHLmvxH4SkQcDBwOnCfpYGACcF9EHAjcl9YBTgIOTI/xwI8qHrWZmbVIk8k/IpZFxONpeS3wLNAbOA24JVW7BTg9LZ8G3BqZh4HukvatdOBmZla+Zs35S+oHHAo8AvSKiGVp0ytAr7TcG1iSa1abyszMrJ0oOflL2hX4b+DCiPhHfltEBBDNObCk8ZLmSpq7YsWK5jQ1M7MWKin5S+pClvhvi4hfpeLlddM56eerqXwp0DfXvE8q20pEXB8RNRFR07Nnz3LjNzOzMpTybh8BNwLPRsT3cptmAKPS8ijgrlz5Z9K7fg4H1uSmh8zMrB0o5cZuHwLOAZ6UND+VfQ2YCEyTNBb4G3Bm2nYvMAJYBLwBjKlkwGZm1nJNJv+ImANoG5uPa6B+AOe1MC4zM6sif8LXzKyAnPzNzArIyd/MrICc/M3MCsjJ38ysgJz8zcwKyMnfzKyAnPzNzArIyd/MrICc/M3MCsjJ38ysgJz8zcwKyMnfzKyAnPzNzArIyd/MrICc/M3MCsjJ38ysgJz8zcwKyMnfzKyAnPzNzAqoyeQv6SZJr0p6Kle2p6Q/SFqYfu6RyiVpsqRFkhZIOqyawZuZWXlKOfOfApxYr2wCcF9EHAjcl9YBTgIOTI/xwI8qE6aZmVVSk8k/ImYDr9UrPg24JS3fApyeK781Mg8D3SXtW6FYzcysQsqd8+8VEcvS8itAr7TcG1iSq1ebyt5G0nhJcyXNXbFiRZlhmJlZOVp8wTciAogy2l0fETURUdOzZ8+WhmFmZs3Qucx2yyXtGxHL0rTOq6l8KdA3V69PKrPtXL8J9zS7zeKJJ1chkrZR9Offnvm1aVi5Z/4zgFFpeRRwV678M+ldP4cDa3LTQ2Zm1k40eeYv6XZgONBDUi3wDWAiME3SWOBvwJmp+r3ACGAR8AYwpgoxbxfKORspRxHOYMys+ZpM/hFx9jY2HddA3QDOa2lQZmZWXeXO+Zttt1rrvzKztuTkb23GF+LM2o7v7WNmVkBO/mZmBeRpH+tQPFVkVhk+8zczKyAnfzOzAnLyNzMrICd/M7MCcvI3MysgJ38zswJy8jczKyC/z9+sjfgzC9aWnPztbba3G5ttb8/HrBI87WNmVkA+89/O+azXzBriM38zswJy8jczKyBP+5h1IO35HULtOTZ7u6okf0knAt8HOgE3RMTEahynKZ7vNmvf/Dvadiqe/CV1Av4LOB6oBR6TNCMinqnogRZMg/uuhDW1sHsfOO5yGHhmRQ9hrefUHeZwSedp7KeVvBw9uHrjmczYfFRbh1VYRUjKRR9z1TjzHwIsiogXASRNBU4DKpf8F0yD31wAG97M1tcsydbBfwA6oFN3mMPELjews/4NQB+tZGKXG2ADhfpltNbT1Jgr949fR5rGqkby7w0sya3XAkMreoT7rnwr8dfZ8GZW7uTf4VzSedqWX8I6O+vfXNJ5GjP+7eRvlVetMdeRrnu02QVfSeOB8Wl1naTnS237wX13+GDDW55l3kWalyvoAawsN8Yqc2zJ6dt8Pdfxt2WnzKtX2F77rb3GhSa139hoo34rccy1Smya1Owm+bjeVe5xq5H8lwJ9c+t9UtlWIuJ64PoqHH8LSXMjoqaaxyiXYytPe42tvcYFjq1c7TW2SsVVjff5PwYcKGl/STsCZwEzqnAcMzMrU8XP/CNio6Tzgd+RvdXzpoh4utLHMTOz8lVlzj8i7gXurca+m6mq00ot5NjK015ja69xgWMrV3uNrSJxKSIqsR8zM+tAfG8fM7MC6tDJX1InSX+RdHeu7CxJX5c0UtICSU9K+rOkQ3J1TpT0vKRFkiZUOKaDJM3PPf4h6cK07XBJP5V0vKR5KbZ5ko7Ntf9gKl8kabIktTCemyS9KumpXNmekv4gaWH6uUduWxdJj0vqK2mmpGckPS3py6W0r2ScqbyUPvtPSUskrWtpHNuIrdxx1uBzqlBMLR1nFe2zKo2zT6SyzZIq8q6bcseZpJ0l3SPpuRRTVW5ZU85Ya6wPGxURHfYBXAT8Arg7V3YL8EHgSGCPVHYS8Eha7gS8ABwA7Ag8ARxcpfg6Aa8A70rr3wTOAA4F9ktl/YGluTaPAocDAv4HOKmFMQwDDgOeypVdDUxIyxOASbltxwA/APYFDktluwF/reunxtpXMs5m9NnhKd517WWcNfac2sk4q2ifVWmcvR84CJgF1FQrzlL6DNgZOCYt7wg80NLfzUqNtcb6sNFjVXNQVvNB9vmB+4Bj6zqKLGE+QbqWkau7R+5FPAL4XW7bpcClVYrxBODB3PocYPd6dQS8BnRNL+JzuW1nAz+pQBz96v1SPg/smxs4z+e2TQJGNLCPu4Djm2pfyThL6bN65RVP/uWOs8aeU1uPs2r1WaXHWa5sFhVK/pUYZ2nb94Fx7WmsNdaHDT068rTPdcAlwOZc2aHAE5F6IGcs2Vk0NHz7id5VivEs4HYAST2ADRGxpl6dM4DHI+JfKY7aVoitV0QsS8uvAL1y244h+2XbQlI/sr59pIT2FVNin1XbdZQ3zlpTc8dZa2npOGsVze0zSd2Bj5Il6kq6jhaOteb0YYe8n7+kU4BXI2KepOG5TSdSr0MkHUPWUa16kxhlH3A7lew/C8jOzn5fr84HyM6ATmjN2PIiIiRFiqc38FpEvFG3XdKuwH8DF0bEPxprXwVt2mceZ5XT0nFWZSX3maTOZH9oJ0e6eWUlVGKsNbcPO2TyBz4EnCppBNANeIekn5P923RGXSVJA4EbyObmVqXikm4/UQEnkZ01LM+tfy8XWx/g18BnIuKFXGx9WiG25ZL2jYhlkvYFXk3lJ5J9OK8uxi5kg+m2iPhVCe0rrZQ+q6aWjLPWUs44ay0tHWetpTl9dj2wMCKuq3AMLRprZfVhJees2uIBDAfuBnYH5uTK3wksAo6sV78z8CKwP29d8P1AFeKaCoyJBubtgO5p/T8aaFf/gu/b5kXLiKUfW8/FfpetL8RdnZZ/CbwvF/OtwHUN7K/B9pWMszl9lmtflQu+5YyzbfV9exln1eizSo+z3H5mUaU5/2b+bl5FlmB3qNbrWc5YK6UPGzxONZ9EazxyHfVx4Ipc+Q3A68D89Jib2zaC7Ir4C8DXqxDTLsAq0gUkoAaYktt+GfDPXGzzgb1zdZ9Ksf2wblC2IJbbgWXABrJrCGOBvcjmKxcCfwT2JHvHyF9y7Y4CAliQi3FE2va29hXos/pxfrUZfXZ1arM5/byipfFUaJy9re/b0TiraJ9VaZx9LO3rX8Bycm/UaO1xRnYGHsCzufJzKz3OyhlrjfVhY4/t5hO+km4g+8rIh9s6lvokXUb2BTdT2zqWbZF0FPDpiPh8W8cC7bfPPM5axuOsdNUea9tN8jczs9J15Ld6mplZmZz8zcwKyMnfzKyAnPzNzArIyd+sGSR1l/TF3Prw/B0YzToKJ3+z5ukOfLGpSmbtnZO/bbck9Uv3X58i6a+SbpP0EUkPpnvMD0n3nL8z3Sf94fTxeSRdke79PkvSi5IuSLudCLxb2T30v5vKdpU0PR3rNin7DgZJE9M91hdIuqYNusBsmzrqvX3MSvUe4BPAZ4HHgE+RfSLyVOBrZHd4/UtEnJ6+uONWYFBq+z6yu0/uBjwv6UdktynoHxGDIJv2IbuL4geAl4EHgQ9JepbsE6rvi4hId4I0azd85m/bu5ci4smI2Aw8DdwX2ScbnyS7x8tRwM8AIuJ+YC9J70ht74mIf0XESrKbkm3r1tWPRkRtOsb8tN81wHrgRkn/AbyxjbZmbcLJ37Z3+Xuxb86tb6bp/3zzbTc1Uv9t9SJiIzAEmA6cAvy21IDNWoOTvxXdA8BI2DKFszIavxf6WrJpoEale6vvHhH3Av8bOKTFkZpVkOf8reiuAG6StIBsamZUY5UjYlW6YPwU2S2379lG1d2AuyR1I7vl7kWVC9ms5XxjNzOzAvK0j5lZATn5m5kVkJO/mVkBOfmbmRWQk7+ZWQE5+ZuZFZCTv5lZATn5m5kV0P8HWeaMFebs8h8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_months(months, lockdown_starts=[\"20200315\", \"20201214\", \"20211219\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lockdown starts: 15-03-2020, 14-12-2020 and 19-12-2020."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>month</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>202004</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>202005</td>\n",
       "      <td>546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>202006</td>\n",
       "      <td>236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>202007</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>202008</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>202009</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>202010</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>202011</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>202012</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>202101</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>202102</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>202103</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>202104</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>202105</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>202106</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>202108</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>202109</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>202110</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>202111</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>202112</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>202201</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>202202</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     month  count\n",
       "0   202004     13\n",
       "1   202005    546\n",
       "2   202006    236\n",
       "3   202007     86\n",
       "4   202008     86\n",
       "5   202009     26\n",
       "6   202010     34\n",
       "7   202011     94\n",
       "8   202012     77\n",
       "9   202101     65\n",
       "10  202102     36\n",
       "11  202103     81\n",
       "12  202104     94\n",
       "13  202105     38\n",
       "14  202106     18\n",
       "15  202108     11\n",
       "16  202109     18\n",
       "17  202110     15\n",
       "18  202111     14\n",
       "19  202112     93\n",
       "20  202201      5\n",
       "21  202202      2"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([{\"month\": key, \"count\": months[key]} for key in sorted(months.keys())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1688\n"
     ]
    }
   ],
   "source": [
    "file_types = get_file_types(file_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file type</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Afbeelding</td>\n",
       "      <td>978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>video</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tekst</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>audio</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Image</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>pdf</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    file type  count\n",
       "0  Afbeelding    978\n",
       "1       video    421\n",
       "2       tekst    106\n",
       "3       audio     92\n",
       "4       Image     67\n",
       "5         pdf     24"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([{\"file type\": key,\n",
    "               \"count\": file_types[key]}\n",
    "              for key in sorted(file_types.keys(), key=lambda k: file_types[k], reverse=True)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## To do\n",
    "\n",
    "* look for author of documents\n",
    "* extract text from documents\n",
    "* counts words per document\n",
    "* determine language of documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python37",
   "language": "python",
   "name": "python37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}