stroll_twitter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8a46ba36-2904-4222-89c4-4d2e6d584948",
   "metadata": {},
   "source": [
    "# Stroll Twitter\n",
    "\n",
    "Use Stroll semantic role labeling for information extraction from tweets."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c881825-49a1-4203-9985-73813a8d8d7f",
   "metadata": {},
   "source": [
    "## 1. Reading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7dfb8655-639a-46d8-86dd-b957a7536665",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d215a8f4-60e9-4040-8d9c-a218c87b43d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIR = \"../../puregome/data/text/\"\n",
    "TEXT = \"text\"\n",
    "USER = \"user\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4fdbbc5-61a9-4d39-95a6-96c09c24b3fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def squeal(text=None):\n",
    "    clear_output(wait=True)\n",
    "    if not text is None: print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "58cea2fa-f15a-4e69-b57f-b94a90c8ceed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_urls(tweet_text):\n",
    "    return(re.sub(r\"\\bhttps?://\\S*\",\"\",tweet_text,flags=re.IGNORECASE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9e3d01be-9b98-4093-8efd-3ec41cfcccaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def restore_newlines(tweet_text):\n",
    "    return re.sub(r'\\\\n', '\\n', tweet_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e7da21e7-eb5e-483f-bce6-ecefa0a50e0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_tweets(file_pattern):\n",
    "    file_names = sorted(os.listdir(DATADIR))\n",
    "    texts = []\n",
    "    for file_name in file_names:\n",
    "        if re.search(file_pattern, file_name):\n",
    "            df = pd.read_csv(DATADIR + file_name)\n",
    "            texts.extend(list(df[TEXT]))\n",
    "            squeal(file_name)\n",
    "    return texts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ca1c0e9-1e09-4a3a-b18c-bff8bdb99f8f",
   "metadata": {},
   "source": [
    "## 2. Stroll\n",
    "\n",
    "Functions copied from stroll_srl_test.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ec7348db-f739-4b96-974b-107458927698",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import stanza\n",
    "import stroll.stanza"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c0112df9-4c38-4848-a338-abd32f83d053",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_text(text):\n",
    "    nlp_analysis = run_nlp(text)\n",
    "    nlp_table_df = nlp_analysis_to_table(nlp_analysis)\n",
    "    nlp_table_df = correct_attachments_table(nlp_table_df)\n",
    "    srl_table_df = nlp_table_to_srl_table(nlp_table_df)\n",
    "    return text, nlp_table_df, srl_table_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b073ae24-9c8c-4f54-8c0e-809d093368d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-01-18 11:13:22 INFO: Loading these models for language: nl (Dutch):\n",
      "=======================\n",
      "| Processor | Package |\n",
      "-----------------------\n",
      "| tokenize  | alpino  |\n",
      "| pos       | alpino  |\n",
      "| lemma     | alpino  |\n",
      "| depparse  | alpino  |\n",
      "| srl       | default |\n",
      "=======================\n",
      "\n",
      "2022-01-18 11:13:22 INFO: Use device: cpu\n",
      "2022-01-18 11:13:22 INFO: Loading: tokenize\n",
      "2022-01-18 11:13:22 INFO: Loading: pos\n",
      "2022-01-18 11:13:22 INFO: Loading: lemma\n",
      "2022-01-18 11:13:22 INFO: Loading: depparse\n",
      "2022-01-18 11:13:23 INFO: Loading: srl\n",
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n",
      "2022-01-18 11:13:24 INFO: Done loading processors!\n"
     ]
    }
   ],
   "source": [
    "run_nlp = stanza.Pipeline(lang='nl', processors='tokenize,lemma,pos,depparse,srl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "65726189-0b26-4950-81b2-ea24ed847d47",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nlp_analysis_to_table(nlp_analysis):\n",
    "    nbr_of_words = 0\n",
    "    for s in nlp_analysis.sentences:\n",
    "        for w in s.words:\n",
    "            if nbr_of_words == 0:\n",
    "                nlp_table_df = pd.DataFrame({\"id\": [w.id], \n",
    "                                             \"text\": [w.text], \n",
    "                                             \"lemma\": [w.lemma],\n",
    "                                             \"upos\": [w.upos],\n",
    "                                             \"xpos\": [w.xpos],\n",
    "                                             \"feats\": [w.feats],\n",
    "                                             \"head\": [w.head],\n",
    "                                             \"deprel\": [w.deprel],\n",
    "                                             \"deps\": [w.deps],\n",
    "                                             \"misc\": [w.misc],\n",
    "                                             \"start_char\": [w.start_char],\n",
    "                                             \"end_char\": [w.end_char],\n",
    "                                             \"parent\": [w.parent],\n",
    "                                             \"sent\": [w.sent],\n",
    "                                             \"srl\": [w.srl],\n",
    "                                             \"frame\": [w.frame],\n",
    "                                            })\n",
    "            else:\n",
    "                nlp_table_df.loc[len(nlp_table_df.index)] = [ w.id, w.text, w.lemma, w.upos, w.xpos, w.feats, \n",
    "                                                              w.head, w.deprel, w.deps, w.misc, w.start_char, w.end_char, \n",
    "                                                              w.parent, w.sent, w.srl, w.frame, ]\n",
    "            nbr_of_words += 1\n",
    "    return nlp_table_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "822eeebd-cd0b-4e0d-9ae7-6d1b670bb947",
   "metadata": {},
   "outputs": [],
   "source": [
    "SRL_FIELDS = [ \"sent_id\", \"head_id\", \"head\", \"nsubj\", \"rel\", \"Arg0\", \"Arg1\", \"Arg2\", \n",
    "               \"ArgM-ADV\", \"ArgM-CAU\", \"ArgM-DIS\", \"ArgM-LOC\", \"ArgM-MNR\", \"ArgM-MOD\", \"ArgM-NEG\", \"ArgM-REC\", \"ArgM-TMP\", ]\n",
    "\n",
    "\n",
    "def srl_dict_to_srl_list(srl_dict, nlp_dict):\n",
    "    srl_list = len(SRL_FIELDS) * [ \"\" ]\n",
    "    for i in range(0, len(SRL_FIELDS)):\n",
    "        if SRL_FIELDS[i] in srl_dict:\n",
    "            srl_list[i] = srl_dict[SRL_FIELDS[i]]\n",
    "        if SRL_FIELDS[i] in nlp_dict:\n",
    "            srl_list[i] = nlp_dict[SRL_FIELDS[i]]\n",
    "    return srl_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba943f0e-8c4e-479f-9c81-396f98339906",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_srl_data_to_srl_table(srl_table_df, srl_data, nlp_data, sentence):\n",
    "    print(srl_data)\n",
    "    for phrase_key in srl_data:\n",
    "        if 'head' in srl_data[phrase_key]:\n",
    "            srl_data[phrase_key][\"head\"] += \" \" + sentence[phrase_key]\n",
    "        elif phrase_key > 0:\n",
    "            srl_data[phrase_key][\"head\"] = sentence[phrase_key]\n",
    "        else:\n",
    "            srl_data[phrase_key][\"head\"] = \"FILLER\"\n",
    "        if phrase_key in nlp_data:\n",
    "            srl_table_df.loc[len(srl_table_df)] = srl_dict_to_srl_list(srl_data[phrase_key], nlp_data[phrase_key])\n",
    "        else:\n",
    "            srl_table_df.loc[len(srl_table_df)] = srl_dict_to_srl_list(srl_data[phrase_key], {})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b42742fa-1e00-48be-a79b-c165a9a6179f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def swap_aux_head(sentence_df, child, head, heads_head):\n",
    "    for i in range(0, len(sentence_df)):\n",
    "        if sentence_df.at[i, \"id\"] == head:\n",
    "            sentence_df.at[i, \"head\"] = child\n",
    "        elif sentence_df.at[i, \"id\"] == child:\n",
    "            sentence_df.at[i, \"head\"] = heads_head\n",
    "        elif sentence_df.at[i, \"head\"] == head:\n",
    "            sentence_df.at[i, \"head\"] = child\n",
    "    return sentence_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a97bd81e-b66b-4c12-936d-d74e18468a56",
   "metadata": {},
   "outputs": [],
   "source": [
    "def correct_attachments_sentence(sentence_df):\n",
    "    children = {}\n",
    "    xpos = {}\n",
    "    upos = {}\n",
    "    text = {}\n",
    "    heads = {}\n",
    "    for i, row in sentence_df.iterrows():\n",
    "        child = row[\"id\"]\n",
    "        head = row[\"head\"]\n",
    "        if head not in children:\n",
    "            children[head] = []\n",
    "        children[head].append(child)\n",
    "        xpos[child] = row[\"xpos\"]\n",
    "        upos[child] = row[\"upos\"]\n",
    "        text[child] = row[\"text\"]\n",
    "        heads[child] = head\n",
    "    for head in children:\n",
    "        if head != 0 and not re.search(\"^WW\", xpos[head]):\n",
    "            for child in children[head]:\n",
    "                if re.search(\"^WW\", xpos[child]) and upos[child] == \"AUX\":\n",
    "                    sentence_df = swap_aux_head(sentence_df, child, head, heads[head])\n",
    "    return sentence_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c0949b08-0d74-445f-9c80-8fd1719c860b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def correct_attachments_table(nlp_table_df):\n",
    "    sentence_df = pd.DataFrame([])\n",
    "    nlp_table_df_out = pd.DataFrame([])\n",
    "    last_id = -1\n",
    "    for i, row in nlp_table_df.iterrows():\n",
    "        if row[\"id\"] < last_id:\n",
    "            new_sentence_df = correct_attachments_sentence(sentence_df)\n",
    "            if len(nlp_table_df_out) == 0:\n",
    "                nlp_table_df_out = new_sentence_df\n",
    "            else:\n",
    "                nlp_table_df_out = pd.concat([nlp_table_df_out, new_sentence_df])\n",
    "            sentence_df = pd.DataFrame([])\n",
    "        sentence_df = sentence_df.append(pd.DataFrame([row]), ignore_index = True)\n",
    "        last_id = row[\"id\"]\n",
    "    if len(sentence_df) > 0:\n",
    "        new_sentence_df = correct_attachments_sentence(sentence_df)\n",
    "        if len(nlp_table_df_out) == 0:\n",
    "            nlp_table_df_out = new_sentence_df\n",
    "        else:\n",
    "            nlp_table_df_out = pd.concat([nlp_table_df_out, new_sentence_df])\n",
    "    return nlp_table_df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3e0eadd7-2334-42b7-a6df-5b2dca3adc88",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nlp_table_to_srl_table(nlp_table_df):\n",
    "    srl_table_df = pd.DataFrame({ field: [] for field in SRL_FIELDS })\n",
    "    srl_data = {}\n",
    "    nlp_data = {}\n",
    "    sentence = {}\n",
    "    last_id = 0\n",
    "    sent_id = 1\n",
    "    for i, row in nlp_table_df.iterrows():\n",
    "        if row['id'] <= last_id:\n",
    "            if len(srl_data) > 0:\n",
    "                add_srl_data_to_srl_table(srl_table_df, srl_data, nlp_data, sentence)\n",
    "            sent_id += 1\n",
    "            srl_data = {}\n",
    "            nlp_data = {}\n",
    "            sentence = {}\n",
    "        if row['srl'] != \"_\":\n",
    "            if row['head'] not in srl_data:\n",
    "                srl_data[row['head']] = { \"sent_id\": sent_id, \"head_id\": row[\"head\"] }\n",
    "            if row['srl'] in srl_data[row['head']]:\n",
    "                print(f\"duplicate role for {row['srl']} [{i}]: {srl_data[row['head']][row['srl']]} and {row['lemma']}\")\n",
    "                srl_data[row['head']][row['srl']] += \" \" + row['lemma']\n",
    "            else:\n",
    "                srl_data[row['head']][row['srl']] = row['lemma']\n",
    "        if row['frame'] == \"rel\":\n",
    "            if row['id'] not in srl_data:\n",
    "                srl_data[row['id']] = { \"sent_id\": sent_id, \"head_id\": row[\"id\"] }\n",
    "            if row['frame'] not in srl_data[row['id']]:\n",
    "                srl_data[row['id']][row['frame']] = row['lemma']\n",
    "            else:\n",
    "                srl_data[row['id']][row['frame']] += \" \" + row['lemma']\n",
    "        if row['deprel'] == \"nsubj\":\n",
    "            if row['head'] not in nlp_data:\n",
    "                nlp_data[row['head']] = { \"sent_id\": sent_id, \"head_id\": row[\"head\"] }\n",
    "            if 'nsubj' in nlp_data[row['head']]:\n",
    "                nlp_data[row['head']][\"nsubj\"] += \" \" + row['lemma']\n",
    "            else:\n",
    "                nlp_data[row['head']][\"nsubj\"] = row['lemma']\n",
    "        if row['deprel'] == \"compound:prt\":\n",
    "            if row['head'] not in nlp_data:\n",
    "                nlp_data[row['head']] = { \"sent_id\": sent_id, \"head_id\": row[\"head\"] }\n",
    "            if 'head' in nlp_data[row['head']]:\n",
    "                nlp_data[row['head']][\"head\"] += \" \" + row['lemma']\n",
    "            else:\n",
    "                nlp_data[row['head']][\"head\"] = row['lemma']\n",
    "        last_id = row['id']\n",
    "        sentence[row['id']] = row['lemma'] \n",
    "    if len(srl_data) > 0:\n",
    "        add_srl_data_to_srl_table(srl_table_df, srl_data, nlp_data, sentence)\n",
    "    return srl_table_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c74440ec-2787-486e-937d-0a2035ab534f",
   "metadata": {},
   "source": [
    "## 3. Who, where, when, what, why and how?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "25118924-693b-4cdd-95f8-7b154be275e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_results(results_dict):\n",
    "    results_df = pd.DataFrame([{ \"key\": key, \"count\": results_dict[key], \"best value\": \"\"} \n",
    "                               for key in sorted(results_dict, key=lambda k: results_dict[k], reverse=True)])\n",
    "    best_keys = get_best_keys(results_df)\n",
    "    for i, row in results_df.iterrows():\n",
    "        if row[\"key\"] in best_keys:\n",
    "            results_df.at[i, \"best value\"] = \"yes\"\n",
    "    return results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cb355f50-ce38-4c50-b6b0-007caa55a81b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_keys(results_df):\n",
    "    best_count = -1\n",
    "    best_keys = []\n",
    "    for i, row in results_df.iterrows():\n",
    "        if row[\"count\"] > best_count:\n",
    "            best_count = row[\"count\"]\n",
    "            best_keys = [row[\"key\"]]\n",
    "        elif row[\"count\"] == best_count:\n",
    "            best_keys.append(row[\"key\"])\n",
    "    case_is_upper = []\n",
    "    for key in best_keys:\n",
    "        case_is_upper.append(re.search(r\"^[A-Z]\", key) != None)\n",
    "    if True in case_is_upper:\n",
    "        best_keys = [ best_keys[i] for i in range(0, len(best_keys)) if case_is_upper[i] ] \n",
    "    return best_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "378fdc52-9c07-4403-b398-b91f889b92cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actors(srl_table_df):\n",
    "    actors = {}\n",
    "    for i, row in srl_table_df.iterrows():\n",
    "        if row[\"Arg0\"] != \"\":\n",
    "            actor = row[\"Arg0\"]\n",
    "        elif row[\"nsubj\"] != \"\":\n",
    "            actor = row[\"nsubj\"]\n",
    "        else:\n",
    "            actor = \"\"\n",
    "        if actor != \"\":\n",
    "            if actor in actors:\n",
    "                actors[actor] += 1\n",
    "            else:\n",
    "                actors[actor] = 1\n",
    "    return actors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "39892b82-6c79-4cee-829c-e8e8ae096e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_locations(srl_table_df):\n",
    "    locations = {}\n",
    "    for i, row in srl_table_df.iterrows():\n",
    "        if row[\"ArgM-LOC\"] != \"\":\n",
    "            location = row[\"ArgM-LOC\"]\n",
    "        else:\n",
    "            location = \"\"\n",
    "        if location != \"\":\n",
    "            if location in locations:\n",
    "                locations[location] += 1\n",
    "            else:\n",
    "                locations[location] = 1\n",
    "    return locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "be1b6ab3-bc20-46e4-ba06-826e97278f48",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_times(srl_table_df):\n",
    "    times = {}\n",
    "    for i, row in srl_table_df.iterrows():\n",
    "        if row[\"ArgM-TMP\"] != \"\":\n",
    "            time = row[\"ArgM-TMP\"]\n",
    "        else:\n",
    "            time = \"\"\n",
    "        if time != \"\":\n",
    "            if time in times:\n",
    "                times[time] += 1\n",
    "            else:\n",
    "                times[time] = 1\n",
    "    return times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "abdb5384-1819-4e1d-8e4f-7fc995741747",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_causes(srl_table_df):\n",
    "    causes = {}\n",
    "    for i, row in srl_table_df.iterrows():\n",
    "        if row[\"ArgM-CAU\"] != \"\":\n",
    "            cause = row[\"ArgM-CAU\"]\n",
    "        else:\n",
    "            cause = \"\"\n",
    "        if cause != \"\":\n",
    "            if cause in causes:\n",
    "                causes[cause] += 1\n",
    "            else:\n",
    "                causes[cause] = 1\n",
    "    return causes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d9a55003-f5c6-4e54-83a2-cee522648e92",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_manners(srl_table_df):\n",
    "    manners = {}\n",
    "    for i, row in srl_table_df.iterrows():\n",
    "        if row[\"ArgM-MNR\"] != \"\":\n",
    "            manner = row[\"ArgM-MNR\"]\n",
    "        else:\n",
    "            manner = \"\"\n",
    "        if manner != \"\":\n",
    "            if manner in manners:\n",
    "                manners[manner] += 1\n",
    "            else:\n",
    "                manners[manner] = 1\n",
    "    return manners"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9211dbdd-6f88-4a99-a1eb-626a1f39fc27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actions(srl_table_df):\n",
    "    actions = {}\n",
    "    for i, row in srl_table_df.iterrows():\n",
    "        if row[\"rel\"] != \"\":\n",
    "            action = row[\"rel\"]\n",
    "        elif row[\"head\"] != \"\":\n",
    "            action = row[\"head\"]\n",
    "        else:\n",
    "            action = \"\"\n",
    "        if action != \"\":\n",
    "            if action in actions:\n",
    "                actions[action] += 1\n",
    "            else:\n",
    "                actions[action] = 1\n",
    "    return actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ed397e98-cef4-4a3d-9cf3-138824c4aa80",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_arguments(srl_table_df, number=1):\n",
    "    argument_name = \"Arg\" + str(number)\n",
    "    arguments = {}\n",
    "    for i, row in srl_table_df.iterrows():\n",
    "        if row[argument_name] != \"\":\n",
    "            argument = row[argument_name]\n",
    "        else:\n",
    "            argument = \"\"\n",
    "        if argument != \"\":\n",
    "            if argument in arguments:\n",
    "                arguments[argument] += 1\n",
    "            else:\n",
    "                arguments[argument] = 1\n",
    "    return arguments"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ce56471-3bd7-4f0d-96c8-d906f775e8d0",
   "metadata": {},
   "source": [
    "## 3. Application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "56f7353e-a910-4c6a-8c0f-ce0c4ff3a40a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20211231-07 16820/16820 @Dapperdikkerdje Dankjewel jij ook een gezond en gelukkig 2022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/erikt/projects/navigatingstories/env/lib/python3.7/site-packages/dgl/base.py:45: DGLWarning: Recommend creating graphs by `dgl.graph(data)` instead of `dgl.DGLGraph(data)`.\n",
      "  return warnings.warn(message, category=category, stacklevel=1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "duplicate role for Arg1 [5]: @Dapperdikkerdje and gezond\n",
      "{2: {'sent_id': 1, 'head_id': 2, 'Arg1': '@Dapperdikkerdje gezond', 'rel': 'dankjewen', 'Arg0': 'jij', 'ArgM-DIS': 'ook'}}\n"
     ]
    }
   ],
   "source": [
    "for hour in [ \"20211231-05\", \"20211231-06\", \"20211231-07\",]:\n",
    "    tweet_texts = read_tweets(hour)\n",
    "    nlp_table_df_all = pd.DataFrame([])\n",
    "    srl_table_df_all = pd.DataFrame([])\n",
    "    counter = 0\n",
    "    for tweet_text in tweet_texts:\n",
    "        try:\n",
    "            counter += 1\n",
    "            if counter % 10 == 0:\n",
    "                squeal(f\"{hour} {counter}/{len(tweet_texts)} \" + tweet_text)\n",
    "            text, new_nlp_table_df, new_srl_table_df = analyze_text(restore_newlines(remove_urls(tweet_text)))\n",
    "            nlp_table_df_all = pd.concat([nlp_table_df_all, new_nlp_table_df])\n",
    "            srl_table_df_all = pd.concat([srl_table_df_all, new_srl_table_df])\n",
    "        except: \n",
    "            pass\n",
    "    srl_table_df_all.to_csv(f\"../data{hour}_srl_table_df_all.csv.gz\", compression=\"gzip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c7f660d9-cf76-4176-9a4b-74dd14976a6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(376978, 35804)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nlp_table_df_all), len(srl_table_df_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "dd9acde7-4749-4eaf-87b3-e3dec31c7d89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>count</th>\n",
       "      <th>best value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>zijn</td>\n",
       "      <td>715</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>hebben</td>\n",
       "      <td>339</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gaan</td>\n",
       "      <td>330</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>doen</td>\n",
       "      <td>185</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>komen</td>\n",
       "      <td>177</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FILLER</td>\n",
       "      <td>169</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>zeggen</td>\n",
       "      <td>148</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>zien</td>\n",
       "      <td>141</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>weten</td>\n",
       "      <td>130</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>worden</td>\n",
       "      <td>122</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      key  count best value\n",
       "0    zijn    715        yes\n",
       "1  hebben    339           \n",
       "2    gaan    330           \n",
       "3    doen    185           \n",
       "4   komen    177           \n",
       "5  FILLER    169           \n",
       "6  zeggen    148           \n",
       "7    zien    141           \n",
       "8   weten    130           \n",
       "9  worden    122           "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_results(get_actions(srl_table_df_all))[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d6b14811-530e-4112-8bd1-dafcaa9a5770",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>count</th>\n",
       "      <th>best value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ik</td>\n",
       "      <td>775</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>je</td>\n",
       "      <td>462</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>we</td>\n",
       "      <td>254</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ze</td>\n",
       "      <td>193</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>die</td>\n",
       "      <td>186</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>hij</td>\n",
       "      <td>82</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>jij</td>\n",
       "      <td>55</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>mens</td>\n",
       "      <td>49</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>u</td>\n",
       "      <td>48</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>jullie</td>\n",
       "      <td>47</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      key  count best value\n",
       "0      ik    775        yes\n",
       "1      je    462           \n",
       "2      we    254           \n",
       "3      ze    193           \n",
       "4     die    186           \n",
       "5     hij     82           \n",
       "6     jij     55           \n",
       "7    mens     49           \n",
       "8       u     48           \n",
       "9  jullie     47           "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_results(get_arguments(srl_table_df_all, number=0))[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc341816-1ef9-4fbb-adcc-427839517e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for i, row in nlp_table_df.iterrows():\n",
    "    if len(data) > 0 and row[\"id\"] < data[-1][\"id\"]:\n",
    "        check(data)\n",
    "        data = []\n",
    "    data.append({\"id\": row[\"id\"], \"pos\": row[\"upos\"], \"lemma\": row[\"lemma\"], \"head\": row[\"head\"]})\n",
    "if len(data) > 0:\n",
    "    check(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30a90e09-79bc-41f0-be12-a6a9aec91822",
   "metadata": {},
   "source": [
    "## 4. Read and process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc4ebedc-d4f3-4f70-ad70-65be2a7fa45c",
   "metadata": {},
   "outputs": [],
   "source": [
    "srl_table_df_all = pd.DataFrame([])\n",
    "for hour in [ \"05\", \"06\", \"07\" ]:\n",
    "    srl_table_df = pd.read_csv(f\"20211231-{hour}_srl_table_df_all.csv.gz\", compression=\"gzip\", index_col=0).fillna(\"\")\n",
    "    srl_table_df_all = pd.concat([srl_table_df_all, srl_table_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5a77cf5-7b3d-42d8-8f3a-7356ad116b5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "srl_table_df_all[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9253429e-f408-4505-b18f-08c262b1795e",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(srl_table_df_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1881e11b-5948-4c6c-9c09-f4af9931c002",
   "metadata": {},
   "outputs": [],
   "source": [
    "srl_table_df_all[srl_table_df_all[\"Arg0\"]==\"Rutte\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9768446-aa0e-4b00-b0a4-0d6883c0a24c",
   "metadata": {},
   "outputs": [],
   "source": [
    "analysis = show_results(get_actors(srl_table_df_all))\n",
    "[ (row[\"key\"], row[\"count\"]) for i, row in analysis.iterrows() if re.search(\"^[A-Z]\", row[\"key\"]) ][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "644f6459-c47c-4e77-898b-edfd2258d668",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stroll",
   "language": "python",
   "name": "stroll"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}