data_cleaning_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine Cleaned Raw Tweets (Before Filtering)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(450464, 17)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean = pd.read_csv('Datasets/jan_tweets_clean.csv', index_col=0)\n",
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(536865, 17)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean = pd.read_csv('Datasets/feb_tweets_clean.csv', index_col=0)\n",
    "feb_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(695958, 17)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean = pd.read_csv('Datasets/mar01_tweets_clean.csv', index_col=0)\n",
    "mar01_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (0,1,3,4,5,6,8,9,10,11,12,15,17) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1741994, 17)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_raw = pd.read_csv('Datasets/march17_tweets.csv', index_col=0)\n",
    "mar17_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1658091, 17)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_raw.dropna(how='all', inplace=True)\n",
    "mar17_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                       0\n",
       "conversation_id          0\n",
       "created_at               0\n",
       "date                     0\n",
       "tweet                    0\n",
       "hashtags            121237\n",
       "user_id             121237\n",
       "username            121237\n",
       "name                121284\n",
       "link                121251\n",
       "retweet             121251\n",
       "nlikes              121237\n",
       "nreplies            121251\n",
       "nretweets           121251\n",
       "quote_url          1523270\n",
       "retweet_id         1658091\n",
       "reply_to            121251\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_raw.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1536793, 17)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_raw.dropna(subset=['name', 'link'], inplace=True)\n",
    "mar17_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1481511, 17)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_raw = mar17_raw.drop_duplicates(subset='tweet')\n",
    "mar17_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1481511, 17)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_raw.reset_index(drop=True, inplace=True)\n",
    "mar17_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2177469, 17)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean = pd.concat([mar01_clean_raw, mar17_raw])\n",
    "mar_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1234267158877818880</td>\n",
       "      <td>1.234266e+18</td>\n",
       "      <td>1583107196000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Wearing a mask on a train track somewhere in B...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.222977e+18</td>\n",
       "      <td>KirkP31369321</td>\n",
       "      <td>Kirks Poutine</td>\n",
       "      <td>https://twitter.com/KirkP31369321/status/12342...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1222977020705091588', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1234267142259953664</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107192000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Were willing to start wearing surgical masks e...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.317655e+09</td>\n",
       "      <td>KCUNLEASHED96</td>\n",
       "      <td>KC UNLEASHED96</td>\n",
       "      <td>https://twitter.com/KCUNLEASHED96/status/12342...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>https://twitter.com/sssniperwolf/status/123426...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1317654776', 'username': 'KCUNLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1234267140955336704</td>\n",
       "      <td>1.234254e+18</td>\n",
       "      <td>1583107192000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>It's target are your lungs, don't give it acce...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.199808e+09</td>\n",
       "      <td>1spark780</td>\n",
       "      <td>Judy</td>\n",
       "      <td>https://twitter.com/1spark780/status/123426714...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3199808368', 'username': '1spark...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1234267136018665472</td>\n",
       "      <td>1.234259e+18</td>\n",
       "      <td>1583107191000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>The new police issue masks might not prevent c...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.931759e+09</td>\n",
       "      <td>hueman_being</td>\n",
       "      <td>Ken Parker</td>\n",
       "      <td>https://twitter.com/hueman_being/status/123426...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3931759154', 'username': 'hueman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1234267124304154625</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107188000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>hey i’m sad so i’m gonna do a face mask and th...</td>\n",
       "      <td>[]</td>\n",
       "      <td>4.252255e+08</td>\n",
       "      <td>beratings</td>\n",
       "      <td>baby bug</td>\n",
       "      <td>https://twitter.com/beratings/status/123426712...</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '425225495', 'username': 'beratin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  conversation_id     created_at        date  \\\n",
       "0  1234267158877818880     1.234266e+18  1583107196000  2020-03-01   \n",
       "1  1234267142259953664     1.234267e+18  1583107192000  2020-03-01   \n",
       "2  1234267140955336704     1.234254e+18  1583107192000  2020-03-01   \n",
       "3  1234267136018665472     1.234259e+18  1583107191000  2020-03-01   \n",
       "4  1234267124304154625     1.234267e+18  1583107188000  2020-03-01   \n",
       "\n",
       "                                               tweet hashtags       user_id  \\\n",
       "0  Wearing a mask on a train track somewhere in B...       []  1.222977e+18   \n",
       "1  Were willing to start wearing surgical masks e...       []  1.317655e+09   \n",
       "2  It's target are your lungs, don't give it acce...       []  3.199808e+09   \n",
       "3  The new police issue masks might not prevent c...       []  3.931759e+09   \n",
       "4  hey i’m sad so i’m gonna do a face mask and th...       []  4.252255e+08   \n",
       "\n",
       "        username            name  \\\n",
       "0  KirkP31369321   Kirks Poutine   \n",
       "1  KCUNLEASHED96  KC UNLEASHED96   \n",
       "2      1spark780            Judy   \n",
       "3   hueman_being      Ken Parker   \n",
       "4      beratings        baby bug   \n",
       "\n",
       "                                                link retweet nlikes  nreplies  \\\n",
       "0  https://twitter.com/KirkP31369321/status/12342...   False      1       0.0   \n",
       "1  https://twitter.com/KCUNLEASHED96/status/12342...   False      0       0.0   \n",
       "2  https://twitter.com/1spark780/status/123426714...   False      0       0.0   \n",
       "3  https://twitter.com/hueman_being/status/123426...   False      0       0.0   \n",
       "4  https://twitter.com/beratings/status/123426712...   False     14       1.0   \n",
       "\n",
       "   nretweets                                          quote_url  retweet_id  \\\n",
       "0        0.0                                                NaN         NaN   \n",
       "1        0.0  https://twitter.com/sssniperwolf/status/123426...         NaN   \n",
       "2        0.0                                                NaN         NaN   \n",
       "3        0.0                                                NaN         NaN   \n",
       "4        0.0                                                NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '1222977020705091588', 'username'...  \n",
       "1  [{'user_id': '1317654776', 'username': 'KCUNLE...  \n",
       "2  [{'user_id': '3199808368', 'username': '1spark...  \n",
       "3  [{'user_id': '3931759154', 'username': 'hueman...  \n",
       "4  [{'user_id': '425225495', 'username': 'beratin...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean.reset_index(drop=True, inplace=True)\n",
    "mar_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2177464</th>\n",
       "      <td>1244836821529440256</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>Biotech VC Bob Nelsen called it right on the c...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.290365e+09</td>\n",
       "      <td>statnews</td>\n",
       "      <td>STAT</td>\n",
       "      <td>https://twitter.com/statnews/status/1244836821...</td>\n",
       "      <td>False</td>\n",
       "      <td>67</td>\n",
       "      <td>1.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3290364847', 'username': 'statne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2177465</th>\n",
       "      <td>1244836820678049792</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>Do you know the right way to dispose of your f...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.314812e+09</td>\n",
       "      <td>AstroRadioNews</td>\n",
       "      <td>Astro Radio News</td>\n",
       "      <td>https://twitter.com/AstroRadioNews/status/1244...</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3314811637', 'username': 'AstroR...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2177466</th>\n",
       "      <td>1244836819528822786</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>Looking to help hospitals? Sew masks.\\n https:...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.514511e+07</td>\n",
       "      <td>KTULNews</td>\n",
       "      <td>Tulsa's Channel 8</td>\n",
       "      <td>https://twitter.com/KTULNews/status/1244836819...</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '15145111', 'username': 'KTULNews'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2177467</th>\n",
       "      <td>1244836819054673920</td>\n",
       "      <td>1.244745e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>How many masks did you manufacture today, ghoul?</td>\n",
       "      <td>[]</td>\n",
       "      <td>9.309958e+17</td>\n",
       "      <td>BaxterBen78</td>\n",
       "      <td>Rooftop Korean</td>\n",
       "      <td>https://twitter.com/BaxterBen78/status/1244836...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '930995839299870722', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2177468</th>\n",
       "      <td>1244836818773778432</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>WATCH: What Manitoba’s Chief Public Health off...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.111004e+07</td>\n",
       "      <td>ctvwinnipeg</td>\n",
       "      <td>CTV News Winnipeg</td>\n",
       "      <td>https://twitter.com/ctvwinnipeg/status/1244836...</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '21110041', 'username': 'ctvwinni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          id  conversation_id     created_at        date  \\\n",
       "2177464  1244836821529440256     1.244837e+18  1585627200000  2020-03-31   \n",
       "2177465  1244836820678049792     1.244837e+18  1585627200000  2020-03-31   \n",
       "2177466  1244836819528822786     1.244837e+18  1585627200000  2020-03-31   \n",
       "2177467  1244836819054673920     1.244745e+18  1585627200000  2020-03-31   \n",
       "2177468  1244836818773778432     1.244837e+18  1585627200000  2020-03-31   \n",
       "\n",
       "                                                     tweet hashtags  \\\n",
       "2177464  Biotech VC Bob Nelsen called it right on the c...       []   \n",
       "2177465  Do you know the right way to dispose of your f...       []   \n",
       "2177466  Looking to help hospitals? Sew masks.\\n https:...       []   \n",
       "2177467   How many masks did you manufacture today, ghoul?       []   \n",
       "2177468  WATCH: What Manitoba’s Chief Public Health off...       []   \n",
       "\n",
       "              user_id        username               name  \\\n",
       "2177464  3.290365e+09        statnews               STAT   \n",
       "2177465  3.314812e+09  AstroRadioNews   Astro Radio News   \n",
       "2177466  1.514511e+07        KTULNews  Tulsa's Channel 8   \n",
       "2177467  9.309958e+17     BaxterBen78     Rooftop Korean   \n",
       "2177468  2.111004e+07     ctvwinnipeg  CTV News Winnipeg   \n",
       "\n",
       "                                                      link retweet nlikes  \\\n",
       "2177464  https://twitter.com/statnews/status/1244836821...   False     67   \n",
       "2177465  https://twitter.com/AstroRadioNews/status/1244...   False      9   \n",
       "2177466  https://twitter.com/KTULNews/status/1244836819...   False      2   \n",
       "2177467  https://twitter.com/BaxterBen78/status/1244836...   False      0   \n",
       "2177468  https://twitter.com/ctvwinnipeg/status/1244836...   False      5   \n",
       "\n",
       "         nreplies  nretweets quote_url  retweet_id  \\\n",
       "2177464       1.0       40.0       NaN         NaN   \n",
       "2177465       0.0        9.0       NaN         NaN   \n",
       "2177466       0.0        0.0       NaN         NaN   \n",
       "2177467       0.0        0.0       NaN         NaN   \n",
       "2177468       2.0        4.0       NaN         NaN   \n",
       "\n",
       "                                                  reply_to  \n",
       "2177464  [{'user_id': '3290364847', 'username': 'statne...  \n",
       "2177465  [{'user_id': '3314811637', 'username': 'AstroR...  \n",
       "2177466  [{'user_id': '15145111', 'username': 'KTULNews'}]  \n",
       "2177467  [{'user_id': '930995839299870722', 'username':...  \n",
       "2177468  [{'user_id': '21110041', 'username': 'ctvwinni...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "mar_clean.to_csv('Datasets/mar_tweets_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3164798, 17)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_thru_mar_clean = pd.concat([jan_clean_raw, feb_clean_raw, mar_clean_raw])\n",
    "jan_thru_mar_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.21252e+18</td>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.57792e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>SpongeBob: slides down Sand Mountain\\nThe tree...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.099210e+18</td>\n",
       "      <td>Matt04181</td>\n",
       "      <td>Matt0417 (CEO of POOP)</td>\n",
       "      <td>https://twitter.com/Matt04181/status/121252388...</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1099209894295678977', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.21252e+18</td>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.57792e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Started 2020 with freshly washed bed sheets, a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>5.777754e+07</td>\n",
       "      <td>standtallx</td>\n",
       "      <td>Emma 🧛🏻‍♀️</td>\n",
       "      <td>https://twitter.com/standtallx/status/12125238...</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '57777541', 'username': 'standtal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.21252e+18</td>\n",
       "      <td>1.212421e+18</td>\n",
       "      <td>1.57792e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Using the coverage of his yellow card to mask ...</td>\n",
       "      <td>['#classicjose', '#coverup']</td>\n",
       "      <td>3.466098e+08</td>\n",
       "      <td>5_Times_LFC</td>\n",
       "      <td>6 Times LFC</td>\n",
       "      <td>https://twitter.com/5_Times_LFC/status/1212523...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '346609834', 'username': '5_Times...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.21252e+18</td>\n",
       "      <td>1.212403e+18</td>\n",
       "      <td>1.57792e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>The mask  pic.twitter.com/vHi9pC9S62</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.070006e+18</td>\n",
       "      <td>kagura_shelby</td>\n",
       "      <td>mugiwara no</td>\n",
       "      <td>https://twitter.com/kagura_shelby/status/12125...</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1070005555777945600', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.21252e+18</td>\n",
       "      <td>1.212513e+18</td>\n",
       "      <td>1.57792e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>I know what you mean, I've been depressed too ...</td>\n",
       "      <td>['#hugs']</td>\n",
       "      <td>4.865904e+07</td>\n",
       "      <td>IBdaSweet1</td>\n",
       "      <td>Luwamba L.J.Taylor</td>\n",
       "      <td>https://twitter.com/IBdaSweet1/status/12125237...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '48659042', 'username': 'IBdaSwee...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id  conversation_id   created_at        date  \\\n",
       "0  1.21252e+18     1.212524e+18  1.57792e+12  2020-01-01   \n",
       "1  1.21252e+18     1.212524e+18  1.57792e+12  2020-01-01   \n",
       "2  1.21252e+18     1.212421e+18  1.57792e+12  2020-01-01   \n",
       "3  1.21252e+18     1.212403e+18  1.57792e+12  2020-01-01   \n",
       "4  1.21252e+18     1.212513e+18  1.57792e+12  2020-01-01   \n",
       "\n",
       "                                               tweet  \\\n",
       "0  SpongeBob: slides down Sand Mountain\\nThe tree...   \n",
       "1  Started 2020 with freshly washed bed sheets, a...   \n",
       "2  Using the coverage of his yellow card to mask ...   \n",
       "3               The mask  pic.twitter.com/vHi9pC9S62   \n",
       "4  I know what you mean, I've been depressed too ...   \n",
       "\n",
       "                       hashtags       user_id       username  \\\n",
       "0                            []  1.099210e+18      Matt04181   \n",
       "1                            []  5.777754e+07     standtallx   \n",
       "2  ['#classicjose', '#coverup']  3.466098e+08    5_Times_LFC   \n",
       "3                            []  1.070006e+18  kagura_shelby   \n",
       "4                     ['#hugs']  4.865904e+07     IBdaSweet1   \n",
       "\n",
       "                     name                                               link  \\\n",
       "0  Matt0417 (CEO of POOP)  https://twitter.com/Matt04181/status/121252388...   \n",
       "1              Emma 🧛🏻‍♀️  https://twitter.com/standtallx/status/12125238...   \n",
       "2             6 Times LFC  https://twitter.com/5_Times_LFC/status/1212523...   \n",
       "3             mugiwara no  https://twitter.com/kagura_shelby/status/12125...   \n",
       "4      Luwamba L.J.Taylor  https://twitter.com/IBdaSweet1/status/12125237...   \n",
       "\n",
       "  retweet nlikes  nreplies  nretweets quote_url  retweet_id  \\\n",
       "0   False      4       0.0        0.0       NaN         NaN   \n",
       "1   False      3       2.0        0.0       NaN         NaN   \n",
       "2   False      0       0.0        0.0       NaN         NaN   \n",
       "3   False      3       0.0        0.0       NaN         NaN   \n",
       "4   False      1       0.0        0.0       NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '1099209894295678977', 'username'...  \n",
       "1  [{'user_id': '57777541', 'username': 'standtal...  \n",
       "2  [{'user_id': '346609834', 'username': '5_Times...  \n",
       "3  [{'user_id': '1070005555777945600', 'username'...  \n",
       "4  [{'user_id': '48659042', 'username': 'IBdaSwee...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_thru_mar_clean.reset_index(drop=True, inplace=True)\n",
    "jan_thru_mar_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3164793</th>\n",
       "      <td>1244836821529440256</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>Biotech VC Bob Nelsen called it right on the c...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.290365e+09</td>\n",
       "      <td>statnews</td>\n",
       "      <td>STAT</td>\n",
       "      <td>https://twitter.com/statnews/status/1244836821...</td>\n",
       "      <td>False</td>\n",
       "      <td>67</td>\n",
       "      <td>1.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3290364847', 'username': 'statne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3164794</th>\n",
       "      <td>1244836820678049792</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>Do you know the right way to dispose of your f...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.314812e+09</td>\n",
       "      <td>AstroRadioNews</td>\n",
       "      <td>Astro Radio News</td>\n",
       "      <td>https://twitter.com/AstroRadioNews/status/1244...</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3314811637', 'username': 'AstroR...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3164795</th>\n",
       "      <td>1244836819528822786</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>Looking to help hospitals? Sew masks.\\n https:...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.514511e+07</td>\n",
       "      <td>KTULNews</td>\n",
       "      <td>Tulsa's Channel 8</td>\n",
       "      <td>https://twitter.com/KTULNews/status/1244836819...</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '15145111', 'username': 'KTULNews'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3164796</th>\n",
       "      <td>1244836819054673920</td>\n",
       "      <td>1.244745e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>How many masks did you manufacture today, ghoul?</td>\n",
       "      <td>[]</td>\n",
       "      <td>9.309958e+17</td>\n",
       "      <td>BaxterBen78</td>\n",
       "      <td>Rooftop Korean</td>\n",
       "      <td>https://twitter.com/BaxterBen78/status/1244836...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '930995839299870722', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3164797</th>\n",
       "      <td>1244836818773778432</td>\n",
       "      <td>1.244837e+18</td>\n",
       "      <td>1585627200000</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>WATCH: What Manitoba’s Chief Public Health off...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.111004e+07</td>\n",
       "      <td>ctvwinnipeg</td>\n",
       "      <td>CTV News Winnipeg</td>\n",
       "      <td>https://twitter.com/ctvwinnipeg/status/1244836...</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '21110041', 'username': 'ctvwinni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          id  conversation_id     created_at        date  \\\n",
       "3164793  1244836821529440256     1.244837e+18  1585627200000  2020-03-31   \n",
       "3164794  1244836820678049792     1.244837e+18  1585627200000  2020-03-31   \n",
       "3164795  1244836819528822786     1.244837e+18  1585627200000  2020-03-31   \n",
       "3164796  1244836819054673920     1.244745e+18  1585627200000  2020-03-31   \n",
       "3164797  1244836818773778432     1.244837e+18  1585627200000  2020-03-31   \n",
       "\n",
       "                                                     tweet hashtags  \\\n",
       "3164793  Biotech VC Bob Nelsen called it right on the c...       []   \n",
       "3164794  Do you know the right way to dispose of your f...       []   \n",
       "3164795  Looking to help hospitals? Sew masks.\\n https:...       []   \n",
       "3164796   How many masks did you manufacture today, ghoul?       []   \n",
       "3164797  WATCH: What Manitoba’s Chief Public Health off...       []   \n",
       "\n",
       "              user_id        username               name  \\\n",
       "3164793  3.290365e+09        statnews               STAT   \n",
       "3164794  3.314812e+09  AstroRadioNews   Astro Radio News   \n",
       "3164795  1.514511e+07        KTULNews  Tulsa's Channel 8   \n",
       "3164796  9.309958e+17     BaxterBen78     Rooftop Korean   \n",
       "3164797  2.111004e+07     ctvwinnipeg  CTV News Winnipeg   \n",
       "\n",
       "                                                      link retweet nlikes  \\\n",
       "3164793  https://twitter.com/statnews/status/1244836821...   False     67   \n",
       "3164794  https://twitter.com/AstroRadioNews/status/1244...   False      9   \n",
       "3164795  https://twitter.com/KTULNews/status/1244836819...   False      2   \n",
       "3164796  https://twitter.com/BaxterBen78/status/1244836...   False      0   \n",
       "3164797  https://twitter.com/ctvwinnipeg/status/1244836...   False      5   \n",
       "\n",
       "         nreplies  nretweets quote_url  retweet_id  \\\n",
       "3164793       1.0       40.0       NaN         NaN   \n",
       "3164794       0.0        9.0       NaN         NaN   \n",
       "3164795       0.0        0.0       NaN         NaN   \n",
       "3164796       0.0        0.0       NaN         NaN   \n",
       "3164797       2.0        4.0       NaN         NaN   \n",
       "\n",
       "                                                  reply_to  \n",
       "3164793  [{'user_id': '3290364847', 'username': 'statne...  \n",
       "3164794  [{'user_id': '3314811637', 'username': 'AstroR...  \n",
       "3164795  [{'user_id': '15145111', 'username': 'KTULNews'}]  \n",
       "3164796  [{'user_id': '930995839299870722', 'username':...  \n",
       "3164797  [{'user_id': '21110041', 'username': 'ctvwinni...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_thru_mar_clean.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "jan_thru_mar_clean.to_csv('Datasets/jan_thru_mar_tweets_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (0,1,3,4,5,6,8,9,10,11,12,15,17) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(306672, 17)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr01_raw = pd.read_csv('Datasets/april01_tweets.csv', index_col=0)\n",
    "apr01_raw.dropna(how='all', inplace=True)\n",
    "apr01_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1209765, 17)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr03_raw = pd.read_csv('Datasets/april03_tweets.csv', index_col=0, engine='python')\n",
    "apr03_raw.dropna(how='all', inplace=True)\n",
    "apr03_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (0,1,3,11,12) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(954701, 17)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr11_raw = pd.read_csv('Datasets/april11_tweets.csv', index_col=0)\n",
    "apr11_raw.dropna(how='all', inplace=True)\n",
    "apr11_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(106920, 17)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr19_raw = pd.read_csv('Datasets/april19_tweets.csv', index_col=0)\n",
    "apr19_raw.dropna(how='all', inplace=True)\n",
    "apr19_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1385734, 17)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr20_raw = pd.read_csv('Datasets/april20_tweets.csv', index_col=0)\n",
    "apr20_raw.dropna(how='all', inplace=True)\n",
    "apr20_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                       0\n",
       "conversation_id          0\n",
       "created_at               0\n",
       "date                     0\n",
       "tweet                    0\n",
       "hashtags             23727\n",
       "user_id              23727\n",
       "username             23728\n",
       "name                 23764\n",
       "link                 23733\n",
       "retweet              23733\n",
       "nlikes               23727\n",
       "nreplies             23733\n",
       "nretweets            23733\n",
       "quote_url          1280006\n",
       "retweet_id         1385734\n",
       "reply_to             23733\n",
       "dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr20_raw.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1361963, 17)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr20_raw.dropna(subset=['name', 'link', 'username', 'retweet'], inplace=True)\n",
    "apr20_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                       0\n",
       "conversation_id          0\n",
       "created_at               0\n",
       "date                     0\n",
       "tweet                    0\n",
       "hashtags                 0\n",
       "user_id                  0\n",
       "username                 0\n",
       "name                     0\n",
       "link                     0\n",
       "retweet                  0\n",
       "nlikes                   0\n",
       "nreplies                 0\n",
       "nretweets                0\n",
       "quote_url          1256242\n",
       "retweet_id         1361963\n",
       "reply_to                 0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr20_raw.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3885771, 17)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean = pd.concat([apr01_raw, apr03_raw, apr11_raw, apr19_raw, apr20_raw])\n",
    "apr_clean.reset_index(drop=True, inplace=True)\n",
    "apr_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1245501194707107842</td>\n",
       "      <td>1.245378e+18</td>\n",
       "      <td>1585785599000</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>.I’m making masks for friends from my fabric s...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.463649e+07</td>\n",
       "      <td>Juliette402</td>\n",
       "      <td>Juliette402</td>\n",
       "      <td>https://twitter.com/Juliette402/status/1245501...</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '24636491', 'username': 'Juliette...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1245501192102465536</td>\n",
       "      <td>1.245500e+18</td>\n",
       "      <td>1585785599000</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>My N95 is going to be autoclaved once a week. ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>8.266323e+17</td>\n",
       "      <td>TyphoidMegs</td>\n",
       "      <td>Typhoid_Megs</td>\n",
       "      <td>https://twitter.com/TyphoidMegs/status/1245501...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '826632321780576259', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1245501189405409280</td>\n",
       "      <td>1.245501e+18</td>\n",
       "      <td>1585785598000</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>ASM @tavarez_f  dropping off masks to the West...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.233065e+18</td>\n",
       "      <td>2665Asds</td>\n",
       "      <td>Rachel ASDS#2665</td>\n",
       "      <td>https://twitter.com/2665Asds/status/1245501189...</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1233064503698436096', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1245501185320202240</td>\n",
       "      <td>1.245501e+18</td>\n",
       "      <td>1585785597000</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>get essential oils and put it in there, it’ll ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.674575e+09</td>\n",
       "      <td>arieldyche</td>\n",
       "      <td>ariel</td>\n",
       "      <td>https://twitter.com/arieldyche/status/12455011...</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1674575418', 'username': 'arield...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1245501185295073282</td>\n",
       "      <td>1.245501e+18</td>\n",
       "      <td>1585785597000</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>I’m super stoked that I can buy a car for chea...</td>\n",
       "      <td>[]</td>\n",
       "      <td>9.333458e+08</td>\n",
       "      <td>TNMouth</td>\n",
       "      <td>Chileman</td>\n",
       "      <td>https://twitter.com/TNMouth/status/12455011852...</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '933345751', 'username': 'TNMouth...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  conversation_id     created_at        date  \\\n",
       "0  1245501194707107842     1.245378e+18  1585785599000  2020-04-01   \n",
       "1  1245501192102465536     1.245500e+18  1585785599000  2020-04-01   \n",
       "2  1245501189405409280     1.245501e+18  1585785598000  2020-04-01   \n",
       "3  1245501185320202240     1.245501e+18  1585785597000  2020-04-01   \n",
       "4  1245501185295073282     1.245501e+18  1585785597000  2020-04-01   \n",
       "\n",
       "                                               tweet hashtags       user_id  \\\n",
       "0  .I’m making masks for friends from my fabric s...       []  2.463649e+07   \n",
       "1  My N95 is going to be autoclaved once a week. ...       []  8.266323e+17   \n",
       "2  ASM @tavarez_f  dropping off masks to the West...       []  1.233065e+18   \n",
       "3  get essential oils and put it in there, it’ll ...       []  1.674575e+09   \n",
       "4  I’m super stoked that I can buy a car for chea...       []  9.333458e+08   \n",
       "\n",
       "      username              name  \\\n",
       "0  Juliette402       Juliette402   \n",
       "1  TyphoidMegs      Typhoid_Megs   \n",
       "2     2665Asds  Rachel ASDS#2665   \n",
       "3   arieldyche             ariel   \n",
       "4      TNMouth          Chileman   \n",
       "\n",
       "                                                link retweet nlikes  nreplies  \\\n",
       "0  https://twitter.com/Juliette402/status/1245501...   False      2       2.0   \n",
       "1  https://twitter.com/TyphoidMegs/status/1245501...   False      1       0.0   \n",
       "2  https://twitter.com/2665Asds/status/1245501189...   False     14       0.0   \n",
       "3  https://twitter.com/arieldyche/status/12455011...   False      3       1.0   \n",
       "4  https://twitter.com/TNMouth/status/12455011852...   False      8       1.0   \n",
       "\n",
       "   nretweets quote_url  retweet_id  \\\n",
       "0        0.0       NaN         NaN   \n",
       "1        0.0       NaN         NaN   \n",
       "2        3.0       NaN         NaN   \n",
       "3        0.0       NaN         NaN   \n",
       "4        3.0       NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '24636491', 'username': 'Juliette...  \n",
       "1  [{'user_id': '826632321780576259', 'username':...  \n",
       "2  [{'user_id': '1233064503698436096', 'username'...  \n",
       "3  [{'user_id': '1674575418', 'username': 'arield...  \n",
       "4  [{'user_id': '933345751', 'username': 'TNMouth...  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3885766</th>\n",
       "      <td>1255708461230436357</td>\n",
       "      <td>1.255708e+18</td>\n",
       "      <td>1588219201000</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>We have been working hard taking care of our c...</td>\n",
       "      <td>['#huntsvilleretirementspecialists', '#huntsvi...</td>\n",
       "      <td>8.706632e+17</td>\n",
       "      <td>Cochran_Mickels</td>\n",
       "      <td>CochranMickels</td>\n",
       "      <td>https://twitter.com/Cochran_Mickels/status/125...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '870663187695370240', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885767</th>\n",
       "      <td>1255708460798357504</td>\n",
       "      <td>1.255708e+18</td>\n",
       "      <td>1588219201000</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>@ZGSIN3 We have KN95 masks in stock in Califor...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.225625e+18</td>\n",
       "      <td>getafacemask</td>\n",
       "      <td>N95 FACE MASKS IN STOCK</td>\n",
       "      <td>https://twitter.com/getafacemask/status/125570...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1225625346508279808', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885768</th>\n",
       "      <td>1255708455408713729</td>\n",
       "      <td>1.255708e+18</td>\n",
       "      <td>1588219200000</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>Wearing a mask or face covering will help flat...</td>\n",
       "      <td>['#stayhomedc']</td>\n",
       "      <td>3.317741e+08</td>\n",
       "      <td>DMHHS_DC</td>\n",
       "      <td>DMHHS #StayHomeDC</td>\n",
       "      <td>https://twitter.com/DMHHS_DC/status/1255708455...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '331774065', 'username': 'DMHHS_D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885769</th>\n",
       "      <td>1255708454725005320</td>\n",
       "      <td>1.255708e+18</td>\n",
       "      <td>1588219200000</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>Drama as cops without face mask try to arrest ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.701397e+09</td>\n",
       "      <td>Nairobi_News</td>\n",
       "      <td>Nairobi News</td>\n",
       "      <td>https://twitter.com/Nairobi_News/status/125570...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1701397302', 'username': 'Nairob...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885770</th>\n",
       "      <td>1255708454704091138</td>\n",
       "      <td>1.255708e+18</td>\n",
       "      <td>1588219200000</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>Starting Monday, all Costco shoppers and emplo...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.734852e+07</td>\n",
       "      <td>StarTribune</td>\n",
       "      <td>Star Tribune</td>\n",
       "      <td>https://twitter.com/StarTribune/status/1255708...</td>\n",
       "      <td>False</td>\n",
       "      <td>42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '17348525', 'username': 'StarTrib...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          id  conversation_id     created_at        date  \\\n",
       "3885766  1255708461230436357     1.255708e+18  1588219201000  2020-04-30   \n",
       "3885767  1255708460798357504     1.255708e+18  1588219201000  2020-04-30   \n",
       "3885768  1255708455408713729     1.255708e+18  1588219200000  2020-04-30   \n",
       "3885769  1255708454725005320     1.255708e+18  1588219200000  2020-04-30   \n",
       "3885770  1255708454704091138     1.255708e+18  1588219200000  2020-04-30   \n",
       "\n",
       "                                                     tweet  \\\n",
       "3885766  We have been working hard taking care of our c...   \n",
       "3885767  @ZGSIN3 We have KN95 masks in stock in Califor...   \n",
       "3885768  Wearing a mask or face covering will help flat...   \n",
       "3885769  Drama as cops without face mask try to arrest ...   \n",
       "3885770  Starting Monday, all Costco shoppers and emplo...   \n",
       "\n",
       "                                                  hashtags       user_id  \\\n",
       "3885766  ['#huntsvilleretirementspecialists', '#huntsvi...  8.706632e+17   \n",
       "3885767                                                 []  1.225625e+18   \n",
       "3885768                                    ['#stayhomedc']  3.317741e+08   \n",
       "3885769                                                 []  1.701397e+09   \n",
       "3885770                                                 []  1.734852e+07   \n",
       "\n",
       "                username                     name  \\\n",
       "3885766  Cochran_Mickels           CochranMickels   \n",
       "3885767     getafacemask  N95 FACE MASKS IN STOCK   \n",
       "3885768         DMHHS_DC        DMHHS #StayHomeDC   \n",
       "3885769     Nairobi_News             Nairobi News   \n",
       "3885770      StarTribune             Star Tribune   \n",
       "\n",
       "                                                      link retweet nlikes  \\\n",
       "3885766  https://twitter.com/Cochran_Mickels/status/125...   False      1   \n",
       "3885767  https://twitter.com/getafacemask/status/125570...   False      0   \n",
       "3885768  https://twitter.com/DMHHS_DC/status/1255708455...   False      0   \n",
       "3885769  https://twitter.com/Nairobi_News/status/125570...   False      1   \n",
       "3885770  https://twitter.com/StarTribune/status/1255708...   False     42   \n",
       "\n",
       "         nreplies  nretweets quote_url  retweet_id  \\\n",
       "3885766       0.0        0.0       NaN         NaN   \n",
       "3885767       1.0        0.0       NaN         NaN   \n",
       "3885768       0.0        0.0       NaN         NaN   \n",
       "3885769       0.0        0.0       NaN         NaN   \n",
       "3885770       5.0        4.0       NaN         NaN   \n",
       "\n",
       "                                                  reply_to  \n",
       "3885766  [{'user_id': '870663187695370240', 'username':...  \n",
       "3885767  [{'user_id': '1225625346508279808', 'username'...  \n",
       "3885768  [{'user_id': '331774065', 'username': 'DMHHS_D...  \n",
       "3885769  [{'user_id': '1701397302', 'username': 'Nairob...  \n",
       "3885770  [{'user_id': '17348525', 'username': 'StarTrib...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "apr_clean.to_csv('Datasets/apr_tweets_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (0,1,3,4,5,6,8,9,10,11,12,15,17) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(567689, 17)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may01_raw = pd.read_csv('Datasets/may01_tweets.csv', index_col=0)\n",
    "may01_raw.dropna(how='all', inplace=True)\n",
    "may01_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1904024, 17)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may05_raw = pd.read_csv('Datasets/may05_tweets.csv', index_col=0, engine='python')\n",
    "may05_raw.dropna(how='all', inplace=True)\n",
    "may05_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (0,1,3,11,12) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1793290, 17)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may18_raw = pd.read_csv('Datasets/may18_tweets.csv', index_col=0)\n",
    "may18_raw.dropna(how='all', inplace=True)\n",
    "may18_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(388012, 17)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may29_raw = pd.read_csv('Datasets/may29_tweets.csv', index_col=0)\n",
    "may29_raw.dropna(how='all', inplace=True)\n",
    "may29_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "conversation_id         0\n",
       "created_at              0\n",
       "date                    0\n",
       "tweet                   0\n",
       "hashtags            21602\n",
       "user_id             21602\n",
       "username            21602\n",
       "name                21616\n",
       "link                21605\n",
       "retweet             21605\n",
       "nlikes              21602\n",
       "nreplies            21605\n",
       "nretweets           21605\n",
       "quote_url          353131\n",
       "retweet_id         388012\n",
       "reply_to            21605\n",
       "dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may29_raw.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(366393, 17)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may29_raw.dropna(subset=['name', 'link', 'username', 'retweet'], inplace=True)\n",
    "may29_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "conversation_id         0\n",
       "created_at              0\n",
       "date                    0\n",
       "tweet                   0\n",
       "hashtags                0\n",
       "user_id                 0\n",
       "username                0\n",
       "name                    0\n",
       "link                    0\n",
       "retweet                 0\n",
       "nlikes                  0\n",
       "nreplies                0\n",
       "nretweets               0\n",
       "quote_url          331516\n",
       "retweet_id         366393\n",
       "reply_to                0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may29_raw.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4584326, 17)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may_clean = pd.concat([may01_raw, may05_raw, may18_raw, may29_raw])\n",
    "may_clean.reset_index(drop=True, inplace=True)\n",
    "may_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1256372829634904064</td>\n",
       "      <td>1.256373e+18</td>\n",
       "      <td>1588377599000</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>🙏🏾❤️🙏🏾Share with those in Need. City Line Chur...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.793749e+08</td>\n",
       "      <td>tsylversrealtor</td>\n",
       "      <td>Tukeitha Sylvers, ABR, E-Pro</td>\n",
       "      <td>https://twitter.com/tsylversrealtor/status/125...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '279374874', 'username': 'tsylver...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1256372827730481152</td>\n",
       "      <td>1.256373e+18</td>\n",
       "      <td>1588377599000</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>Get my art printed on awesome products. Suppor...</td>\n",
       "      <td>['#rbandme', '#findyourthing', '#redbubble', '...</td>\n",
       "      <td>1.942613e+09</td>\n",
       "      <td>NKDarttowear</td>\n",
       "      <td>Nicole Kiefer</td>\n",
       "      <td>https://twitter.com/NKDarttowear/status/125637...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1942613162', 'username': 'NKDart...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1256372824387665920</td>\n",
       "      <td>1.256373e+18</td>\n",
       "      <td>1588377598000</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>Be the hero you were meant to be. Wear a mask....</td>\n",
       "      <td>['#wearamask', '#stayathomeandstaysafe', '#cov...</td>\n",
       "      <td>3.166069e+09</td>\n",
       "      <td>CYC_Coolidge</td>\n",
       "      <td>👬CYC👭</td>\n",
       "      <td>https://twitter.com/CYC_Coolidge/status/125637...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>https://twitter.com/ICSTucson/status/125635613...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3166068824', 'username': 'CYC_Co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1256372822638821376</td>\n",
       "      <td>1.256373e+18</td>\n",
       "      <td>1588377597000</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>Hey Mask Squad Fans! Please support our Missio...</td>\n",
       "      <td>['#masks4all', '#masksquad', '#ppe', '#somerse...</td>\n",
       "      <td>1.256247e+18</td>\n",
       "      <td>MaskSquadNJ</td>\n",
       "      <td>The Mask Squad Somerset NJ</td>\n",
       "      <td>https://twitter.com/MaskSquadNJ/status/1256372...</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1256246961876930563', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1256372818679226371</td>\n",
       "      <td>1.256367e+18</td>\n",
       "      <td>1588377596000</td>\n",
       "      <td>2020-05-01</td>\n",
       "      <td>Those people you say being cautious ,the major...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.167661e+09</td>\n",
       "      <td>BestSon99</td>\n",
       "      <td>LoveTheBeach</td>\n",
       "      <td>https://twitter.com/BestSon99/status/125637281...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2167661227', 'username': 'BestSo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  conversation_id     created_at        date  \\\n",
       "0  1256372829634904064     1.256373e+18  1588377599000  2020-05-01   \n",
       "1  1256372827730481152     1.256373e+18  1588377599000  2020-05-01   \n",
       "2  1256372824387665920     1.256373e+18  1588377598000  2020-05-01   \n",
       "3  1256372822638821376     1.256373e+18  1588377597000  2020-05-01   \n",
       "4  1256372818679226371     1.256367e+18  1588377596000  2020-05-01   \n",
       "\n",
       "                                               tweet  \\\n",
       "0  🙏🏾❤️🙏🏾Share with those in Need. City Line Chur...   \n",
       "1  Get my art printed on awesome products. Suppor...   \n",
       "2  Be the hero you were meant to be. Wear a mask....   \n",
       "3  Hey Mask Squad Fans! Please support our Missio...   \n",
       "4  Those people you say being cautious ,the major...   \n",
       "\n",
       "                                            hashtags       user_id  \\\n",
       "0                                                 []  2.793749e+08   \n",
       "1  ['#rbandme', '#findyourthing', '#redbubble', '...  1.942613e+09   \n",
       "2  ['#wearamask', '#stayathomeandstaysafe', '#cov...  3.166069e+09   \n",
       "3  ['#masks4all', '#masksquad', '#ppe', '#somerse...  1.256247e+18   \n",
       "4                                                 []  2.167661e+09   \n",
       "\n",
       "          username                          name  \\\n",
       "0  tsylversrealtor  Tukeitha Sylvers, ABR, E-Pro   \n",
       "1     NKDarttowear                 Nicole Kiefer   \n",
       "2     CYC_Coolidge                         👬CYC👭   \n",
       "3      MaskSquadNJ    The Mask Squad Somerset NJ   \n",
       "4        BestSon99                  LoveTheBeach   \n",
       "\n",
       "                                                link retweet nlikes  nreplies  \\\n",
       "0  https://twitter.com/tsylversrealtor/status/125...   False      0       0.0   \n",
       "1  https://twitter.com/NKDarttowear/status/125637...   False      0       0.0   \n",
       "2  https://twitter.com/CYC_Coolidge/status/125637...   False      0       0.0   \n",
       "3  https://twitter.com/MaskSquadNJ/status/1256372...   False      2       0.0   \n",
       "4  https://twitter.com/BestSon99/status/125637281...   False      0       0.0   \n",
       "\n",
       "   nretweets                                          quote_url  retweet_id  \\\n",
       "0        0.0                                                NaN         NaN   \n",
       "1        0.0                                                NaN         NaN   \n",
       "2        0.0  https://twitter.com/ICSTucson/status/125635613...         NaN   \n",
       "3        1.0                                                NaN         NaN   \n",
       "4        0.0                                                NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '279374874', 'username': 'tsylver...  \n",
       "1  [{'user_id': '1942613162', 'username': 'NKDart...  \n",
       "2  [{'user_id': '3166068824', 'username': 'CYC_Co...  \n",
       "3  [{'user_id': '1256246961876930563', 'username'...  \n",
       "4  [{'user_id': '2167661227', 'username': 'BestSo...  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4584321</th>\n",
       "      <td>1266942490109771777</td>\n",
       "      <td>1.266902e+18</td>\n",
       "      <td>1590897602000</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>Explain to me then how me and my friends organ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>4.534313e+09</td>\n",
       "      <td>OUAodonoghue</td>\n",
       "      <td>𝙆𝙖𝙩𝙞 𝙨𝙖𝙬 𝙋𝙧𝙞𝙢𝙖 𝘿𝙤𝙣𝙣𝙖 ‎⎊</td>\n",
       "      <td>https://twitter.com/OUAodonoghue/status/126694...</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '4534313234', 'username': 'OUAodo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4584322</th>\n",
       "      <td>1266942487740010497</td>\n",
       "      <td>1.266940e+18</td>\n",
       "      <td>1590897602000</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>Democrat Logic: Requiring the wearing of Masks...</td>\n",
       "      <td>[]</td>\n",
       "      <td>7.965132e+17</td>\n",
       "      <td>American1959</td>\n",
       "      <td>Steve Wirt</td>\n",
       "      <td>https://twitter.com/American1959/status/126694...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '796513206185455616', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4584323</th>\n",
       "      <td>1266942486108200961</td>\n",
       "      <td>1.266666e+18</td>\n",
       "      <td>1590897601000</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>It definitely was peaceful at the Statehouse. ...</td>\n",
       "      <td>['#socialdistancing']</td>\n",
       "      <td>2.502993e+09</td>\n",
       "      <td>LancelotLinkSpy</td>\n",
       "      <td>Lance Link Resists 🙈🙉🙊</td>\n",
       "      <td>https://twitter.com/LancelotLinkSpy/status/126...</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2502992592', 'username': 'Lancel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4584324</th>\n",
       "      <td>1266942481519779845</td>\n",
       "      <td>1.266922e+18</td>\n",
       "      <td>1590897600000</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>I see the video and see two things, aggressive...</td>\n",
       "      <td>[]</td>\n",
       "      <td>7.109589e+17</td>\n",
       "      <td>McCraftBrew</td>\n",
       "      <td>OutsidEverday</td>\n",
       "      <td>https://twitter.com/McCraftBrew/status/1266942...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '710958900342722560', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4584325</th>\n",
       "      <td>1266942481175736320</td>\n",
       "      <td>1.266942e+18</td>\n",
       "      <td>1590897600000</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>New stuff now available on our site! Restock o...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.196503e+18</td>\n",
       "      <td>himawariiLA</td>\n",
       "      <td>himawarii.LA</td>\n",
       "      <td>https://twitter.com/himawariiLA/status/1266942...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1196503025122041856', 'username'...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          id  conversation_id     created_at        date  \\\n",
       "4584321  1266942490109771777     1.266902e+18  1590897602000  2020-05-31   \n",
       "4584322  1266942487740010497     1.266940e+18  1590897602000  2020-05-31   \n",
       "4584323  1266942486108200961     1.266666e+18  1590897601000  2020-05-31   \n",
       "4584324  1266942481519779845     1.266922e+18  1590897600000  2020-05-31   \n",
       "4584325  1266942481175736320     1.266942e+18  1590897600000  2020-05-31   \n",
       "\n",
       "                                                     tweet  \\\n",
       "4584321  Explain to me then how me and my friends organ...   \n",
       "4584322  Democrat Logic: Requiring the wearing of Masks...   \n",
       "4584323  It definitely was peaceful at the Statehouse. ...   \n",
       "4584324  I see the video and see two things, aggressive...   \n",
       "4584325  New stuff now available on our site! Restock o...   \n",
       "\n",
       "                      hashtags       user_id         username  \\\n",
       "4584321                     []  4.534313e+09     OUAodonoghue   \n",
       "4584322                     []  7.965132e+17     American1959   \n",
       "4584323  ['#socialdistancing']  2.502993e+09  LancelotLinkSpy   \n",
       "4584324                     []  7.109589e+17      McCraftBrew   \n",
       "4584325                     []  1.196503e+18      himawariiLA   \n",
       "\n",
       "                            name  \\\n",
       "4584321  𝙆𝙖𝙩𝙞 𝙨𝙖𝙬 𝙋𝙧𝙞𝙢𝙖 𝘿𝙤𝙣𝙣𝙖 ‎⎊   \n",
       "4584322               Steve Wirt   \n",
       "4584323   Lance Link Resists 🙈🙉🙊   \n",
       "4584324            OutsidEverday   \n",
       "4584325             himawarii.LA   \n",
       "\n",
       "                                                      link retweet nlikes  \\\n",
       "4584321  https://twitter.com/OUAodonoghue/status/126694...   False      8   \n",
       "4584322  https://twitter.com/American1959/status/126694...   False      0   \n",
       "4584323  https://twitter.com/LancelotLinkSpy/status/126...   False      2   \n",
       "4584324  https://twitter.com/McCraftBrew/status/1266942...   False      0   \n",
       "4584325  https://twitter.com/himawariiLA/status/1266942...   False      0   \n",
       "\n",
       "         nreplies  nretweets quote_url  retweet_id  \\\n",
       "4584321       2.0        0.0       NaN         NaN   \n",
       "4584322       0.0        0.0       NaN         NaN   \n",
       "4584323       0.0        2.0       NaN         NaN   \n",
       "4584324       3.0        0.0       NaN         NaN   \n",
       "4584325       0.0        0.0       NaN         NaN   \n",
       "\n",
       "                                                  reply_to  \n",
       "4584321  [{'user_id': '4534313234', 'username': 'OUAodo...  \n",
       "4584322  [{'user_id': '796513206185455616', 'username':...  \n",
       "4584323  [{'user_id': '2502992592', 'username': 'Lancel...  \n",
       "4584324  [{'user_id': '710958900342722560', 'username':...  \n",
       "4584325  [{'user_id': '1196503025122041856', 'username'...  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "may_clean.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "may_clean.to_csv('Datasets/may_tweets_clean.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (1,3,11,12) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1212523886329380865</td>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1577923196000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>SpongeBob: slides down Sand Mountain\\nThe tree...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.099210e+18</td>\n",
       "      <td>Matt04181</td>\n",
       "      <td>Matt0417 (CEO of POOP)</td>\n",
       "      <td>https://twitter.com/Matt04181/status/121252388...</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1099209894295678977', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1212523880558026753</td>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1577923195000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Started 2020 with freshly washed bed sheets, a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>5.777754e+07</td>\n",
       "      <td>standtallx</td>\n",
       "      <td>Emma 🧛🏻‍♀️</td>\n",
       "      <td>https://twitter.com/standtallx/status/12125238...</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '57777541', 'username': 'standtal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1212523828888449024</td>\n",
       "      <td>1.212421e+18</td>\n",
       "      <td>1577923183000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Using the coverage of his yellow card to mask ...</td>\n",
       "      <td>['#classicjose', '#coverup']</td>\n",
       "      <td>3.466098e+08</td>\n",
       "      <td>5_Times_LFC</td>\n",
       "      <td>6 Times LFC</td>\n",
       "      <td>https://twitter.com/5_Times_LFC/status/1212523...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '346609834', 'username': '5_Times...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1212523807128186882</td>\n",
       "      <td>1.212403e+18</td>\n",
       "      <td>1577923177000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>The mask  pic.twitter.com/vHi9pC9S62</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.070006e+18</td>\n",
       "      <td>kagura_shelby</td>\n",
       "      <td>mugiwara no</td>\n",
       "      <td>https://twitter.com/kagura_shelby/status/12125...</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1070005555777945600', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1212523752925229056</td>\n",
       "      <td>1.212513e+18</td>\n",
       "      <td>1577923164000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>I know what you mean, I've been depressed too ...</td>\n",
       "      <td>['#hugs']</td>\n",
       "      <td>4.865904e+07</td>\n",
       "      <td>IBdaSweet1</td>\n",
       "      <td>Luwamba L.J.Taylor</td>\n",
       "      <td>https://twitter.com/IBdaSweet1/status/12125237...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '48659042', 'username': 'IBdaSwee...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  conversation_id     created_at        date  \\\n",
       "0  1212523886329380865     1.212524e+18  1577923196000  2020-01-01   \n",
       "1  1212523880558026753     1.212524e+18  1577923195000  2020-01-01   \n",
       "2  1212523828888449024     1.212421e+18  1577923183000  2020-01-01   \n",
       "3  1212523807128186882     1.212403e+18  1577923177000  2020-01-01   \n",
       "4  1212523752925229056     1.212513e+18  1577923164000  2020-01-01   \n",
       "\n",
       "                                               tweet  \\\n",
       "0  SpongeBob: slides down Sand Mountain\\nThe tree...   \n",
       "1  Started 2020 with freshly washed bed sheets, a...   \n",
       "2  Using the coverage of his yellow card to mask ...   \n",
       "3               The mask  pic.twitter.com/vHi9pC9S62   \n",
       "4  I know what you mean, I've been depressed too ...   \n",
       "\n",
       "                       hashtags       user_id       username  \\\n",
       "0                            []  1.099210e+18      Matt04181   \n",
       "1                            []  5.777754e+07     standtallx   \n",
       "2  ['#classicjose', '#coverup']  3.466098e+08    5_Times_LFC   \n",
       "3                            []  1.070006e+18  kagura_shelby   \n",
       "4                     ['#hugs']  4.865904e+07     IBdaSweet1   \n",
       "\n",
       "                     name                                               link  \\\n",
       "0  Matt0417 (CEO of POOP)  https://twitter.com/Matt04181/status/121252388...   \n",
       "1              Emma 🧛🏻‍♀️  https://twitter.com/standtallx/status/12125238...   \n",
       "2             6 Times LFC  https://twitter.com/5_Times_LFC/status/1212523...   \n",
       "3             mugiwara no  https://twitter.com/kagura_shelby/status/12125...   \n",
       "4      Luwamba L.J.Taylor  https://twitter.com/IBdaSweet1/status/12125237...   \n",
       "\n",
       "  retweet nlikes  nreplies  nretweets quote_url  retweet_id  \\\n",
       "0   False      4       0.0        0.0       NaN         NaN   \n",
       "1   False      3       2.0        0.0       NaN         NaN   \n",
       "2   False      0       0.0        0.0       NaN         NaN   \n",
       "3   False      3       0.0        0.0       NaN         NaN   \n",
       "4   False      1       0.0        0.0       NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '1099209894295678977', 'username'...  \n",
       "1  [{'user_id': '57777541', 'username': 'standtal...  \n",
       "2  [{'user_id': '346609834', 'username': '5_Times...  \n",
       "3  [{'user_id': '1070005555777945600', 'username'...  \n",
       "4  [{'user_id': '48659042', 'username': 'IBdaSwee...  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan = pd.read_csv('Datasets/january_tweets.csv', index_col=0)\n",
    "jan.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-01-01', '2020-01-02', '2020-01-03', '2020-01-04',\n",
       "       'Dream Doctor®', '2020-01-05', '2020-01-06', '2020-01-07',\n",
       "       '2020-01-08', nan, 'We The Curators', '2020-01-09', '2020-01-10',\n",
       "       '2020-01-11', '2020-01-12', '2020-01-13', '2020-01-14',\n",
       "       '2020-01-15', '2020-01-16', '2020-01-17', '2020-01-18',\n",
       "       '2020-01-19', '2020-01-20', '2020-01-21', '2020-01-22',\n",
       "       '2020-01-23', '2020-01-24', '2020-01-25', '2020-01-26',\n",
       "       '2020-01-27', '2020-01-28', '2020-01-29', '2020-01-30',\n",
       "       '2020-01-31'], dtype=object)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(487435, 17)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                  10674\n",
       "conversation_id     10674\n",
       "created_at          10674\n",
       "date                10674\n",
       "tweet               10674\n",
       "hashtags            21349\n",
       "user_id             21349\n",
       "username            21349\n",
       "name                21365\n",
       "link                21351\n",
       "retweet             21351\n",
       "nlikes              21349\n",
       "nreplies            21351\n",
       "nretweets           21351\n",
       "quote_url          446776\n",
       "retweet_id         487435\n",
       "reply_to            21351\n",
       "dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (1,3,4,5,6,8,9,10,11,12,15,17) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1223757879305367552</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601589000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>Bought Bruno Fernandes, play him at CDM? Wtf a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.221553e+09</td>\n",
       "      <td>chong_wh96</td>\n",
       "      <td>Chong_wH</td>\n",
       "      <td>https://twitter.com/chong_wh96/status/12237578...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2221552537', 'username': 'chong_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1223757878827159552</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601589000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>My sister sent me $90 worth of face masks and ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.075768e+09</td>\n",
       "      <td>Yamanik30</td>\n",
       "      <td>ACAB</td>\n",
       "      <td>https://twitter.com/Yamanik30/status/122375787...</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3075767827', 'username': 'Yamani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1223757871751426049</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601587000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>Because Nigeria is currently 51% Muslim and th...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.184187e+07</td>\n",
       "      <td>Dimplez</td>\n",
       "      <td>Dimplez 🇳🇬</td>\n",
       "      <td>https://twitter.com/Dimplez/status/12237578717...</td>\n",
       "      <td>False</td>\n",
       "      <td>16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>https://twitter.com/JesseLehrich/status/122336...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '21841872', 'username': 'Dimplez'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1223757866911326209</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601586000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>When you were a mask to stop the cronavirus bu...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.209989e+18</td>\n",
       "      <td>Harry31348302</td>\n",
       "      <td>Harry</td>\n",
       "      <td>https://twitter.com/Harry31348302/status/12237...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1209988745790918656', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1223757863627018241</td>\n",
       "      <td>1.223757e+18</td>\n",
       "      <td>1580601585000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>This 'MeshGraphic' implementation supports:\\n\\...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.478940e+09</td>\n",
       "      <td>Softdrink_117</td>\n",
       "      <td>Softdrink 117</td>\n",
       "      <td>https://twitter.com/Softdrink_117/status/12237...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1478940204', 'username': 'Softdr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  conversation_id     created_at        date  \\\n",
       "0  1223757879305367552     1.223758e+18  1580601589000  2020-02-01   \n",
       "1  1223757878827159552     1.223758e+18  1580601589000  2020-02-01   \n",
       "2  1223757871751426049     1.223758e+18  1580601587000  2020-02-01   \n",
       "3  1223757866911326209     1.223758e+18  1580601586000  2020-02-01   \n",
       "4  1223757863627018241     1.223757e+18  1580601585000  2020-02-01   \n",
       "\n",
       "                                               tweet hashtags       user_id  \\\n",
       "0  Bought Bruno Fernandes, play him at CDM? Wtf a...       []  2.221553e+09   \n",
       "1  My sister sent me $90 worth of face masks and ...       []  3.075768e+09   \n",
       "2  Because Nigeria is currently 51% Muslim and th...       []  2.184187e+07   \n",
       "3  When you were a mask to stop the cronavirus bu...       []  1.209989e+18   \n",
       "4  This 'MeshGraphic' implementation supports:\\n\\...       []  1.478940e+09   \n",
       "\n",
       "        username           name  \\\n",
       "0     chong_wh96       Chong_wH   \n",
       "1      Yamanik30           ACAB   \n",
       "2        Dimplez     Dimplez 🇳🇬   \n",
       "3  Harry31348302          Harry   \n",
       "4  Softdrink_117  Softdrink 117   \n",
       "\n",
       "                                                link retweet nlikes  nreplies  \\\n",
       "0  https://twitter.com/chong_wh96/status/12237578...   False      0       0.0   \n",
       "1  https://twitter.com/Yamanik30/status/122375787...   False      6       0.0   \n",
       "2  https://twitter.com/Dimplez/status/12237578717...   False     16       0.0   \n",
       "3  https://twitter.com/Harry31348302/status/12237...   False      0       0.0   \n",
       "4  https://twitter.com/Softdrink_117/status/12237...   False      0       0.0   \n",
       "\n",
       "   nretweets                                          quote_url  retweet_id  \\\n",
       "0        0.0                                                NaN         NaN   \n",
       "1        0.0                                                NaN         NaN   \n",
       "2        3.0  https://twitter.com/JesseLehrich/status/122336...         NaN   \n",
       "3        0.0                                                NaN         NaN   \n",
       "4        0.0                                                NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '2221552537', 'username': 'chong_...  \n",
       "1  [{'user_id': '3075767827', 'username': 'Yamani...  \n",
       "2   [{'user_id': '21841872', 'username': 'Dimplez'}]  \n",
       "3  [{'user_id': '1209988745790918656', 'username'...  \n",
       "4  [{'user_id': '1478940204', 'username': 'Softdr...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb = pd.read_csv('Datasets/february_tweets.csv', index_col=0)\n",
    "feb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(626691, 17)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                  42530\n",
       "conversation_id     42530\n",
       "created_at          42530\n",
       "date                42530\n",
       "tweet               42530\n",
       "hashtags            72125\n",
       "user_id             72125\n",
       "username            72127\n",
       "name                72143\n",
       "link                72131\n",
       "retweet             72131\n",
       "nlikes              72125\n",
       "nreplies            72131\n",
       "nretweets           72131\n",
       "quote_url          581219\n",
       "retweet_id         626691\n",
       "reply_to            72131\n",
       "dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-02-01', '2020-02-02', '2020-02-03', '2020-02-04',\n",
       "       '2020-02-05', '2020-02-06', '2020-02-07', '2020-02-08',\n",
       "       '2020-02-09', 'jack', '2020-02-10', '2020-02-11', '2020-02-12',\n",
       "       '2020-02-13', 'KennyWonKenOhBe', '2020-02-14', '2020-02-15',\n",
       "       '2020-02-16', '2020-02-17', 'AsiaNews EN', '2020-02-18', 'Femina',\n",
       "       '2020-02-19', '2020-02-20', '2020-02-21', '2020-02-22',\n",
       "       '2020-02-23', nan, 'We The Curators', '2020-02-24', '2020-02-25',\n",
       "       '2020-02-26', '2020-02-27', '2020-02-28', 'Vivian Kwan',\n",
       "       '2020-02-29'], dtype=object)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (1,3,11,12) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1234267158877818880</td>\n",
       "      <td>1.234266e+18</td>\n",
       "      <td>1583107196000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Wearing a mask on a train track somewhere in B...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.222977e+18</td>\n",
       "      <td>KirkP31369321</td>\n",
       "      <td>Kirks Poutine</td>\n",
       "      <td>https://twitter.com/KirkP31369321/status/12342...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1222977020705091588', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1234267142259953664</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107192000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Were willing to start wearing surgical masks e...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.317655e+09</td>\n",
       "      <td>KCUNLEASHED96</td>\n",
       "      <td>KC UNLEASHED96</td>\n",
       "      <td>https://twitter.com/KCUNLEASHED96/status/12342...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>https://twitter.com/sssniperwolf/status/123426...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1317654776', 'username': 'KCUNLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1234267140955336704</td>\n",
       "      <td>1.234254e+18</td>\n",
       "      <td>1583107192000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>It's target are your lungs, don't give it acce...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.199808e+09</td>\n",
       "      <td>1spark780</td>\n",
       "      <td>Judy</td>\n",
       "      <td>https://twitter.com/1spark780/status/123426714...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3199808368', 'username': '1spark...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1234267136018665472</td>\n",
       "      <td>1.234259e+18</td>\n",
       "      <td>1583107191000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>The new police issue masks might not prevent c...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.931759e+09</td>\n",
       "      <td>hueman_being</td>\n",
       "      <td>Ken Parker</td>\n",
       "      <td>https://twitter.com/hueman_being/status/123426...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3931759154', 'username': 'hueman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1234267124304154625</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107188000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>hey i’m sad so i’m gonna do a face mask and th...</td>\n",
       "      <td>[]</td>\n",
       "      <td>4.252255e+08</td>\n",
       "      <td>beratings</td>\n",
       "      <td>baby bug</td>\n",
       "      <td>https://twitter.com/beratings/status/123426712...</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '425225495', 'username': 'beratin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  conversation_id     created_at        date  \\\n",
       "0  1234267158877818880     1.234266e+18  1583107196000  2020-03-01   \n",
       "1  1234267142259953664     1.234267e+18  1583107192000  2020-03-01   \n",
       "2  1234267140955336704     1.234254e+18  1583107192000  2020-03-01   \n",
       "3  1234267136018665472     1.234259e+18  1583107191000  2020-03-01   \n",
       "4  1234267124304154625     1.234267e+18  1583107188000  2020-03-01   \n",
       "\n",
       "                                               tweet hashtags       user_id  \\\n",
       "0  Wearing a mask on a train track somewhere in B...       []  1.222977e+18   \n",
       "1  Were willing to start wearing surgical masks e...       []  1.317655e+09   \n",
       "2  It's target are your lungs, don't give it acce...       []  3.199808e+09   \n",
       "3  The new police issue masks might not prevent c...       []  3.931759e+09   \n",
       "4  hey i’m sad so i’m gonna do a face mask and th...       []  4.252255e+08   \n",
       "\n",
       "        username            name  \\\n",
       "0  KirkP31369321   Kirks Poutine   \n",
       "1  KCUNLEASHED96  KC UNLEASHED96   \n",
       "2      1spark780            Judy   \n",
       "3   hueman_being      Ken Parker   \n",
       "4      beratings        baby bug   \n",
       "\n",
       "                                                link retweet nlikes  nreplies  \\\n",
       "0  https://twitter.com/KirkP31369321/status/12342...   False      1       0.0   \n",
       "1  https://twitter.com/KCUNLEASHED96/status/12342...   False      0       0.0   \n",
       "2  https://twitter.com/1spark780/status/123426714...   False      0       0.0   \n",
       "3  https://twitter.com/hueman_being/status/123426...   False      0       0.0   \n",
       "4  https://twitter.com/beratings/status/123426712...   False     14       1.0   \n",
       "\n",
       "   nretweets                                          quote_url  retweet_id  \\\n",
       "0        0.0                                                NaN         NaN   \n",
       "1        0.0  https://twitter.com/sssniperwolf/status/123426...         NaN   \n",
       "2        0.0                                                NaN         NaN   \n",
       "3        0.0                                                NaN         NaN   \n",
       "4        0.0                                                NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '1222977020705091588', 'username'...  \n",
       "1  [{'user_id': '1317654776', 'username': 'KCUNLE...  \n",
       "2  [{'user_id': '3199808368', 'username': '1spark...  \n",
       "3  [{'user_id': '3931759154', 'username': 'hueman...  \n",
       "4  [{'user_id': '425225495', 'username': 'beratin...  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01 = pd.read_csv('Datasets/march01_tweets.csv', index_col=0)\n",
    "mar01.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(737002, 17)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (0,1,3,4,5,6,8,9,10,11,12,15,17) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(334598, 17)"
      ]
     },
     "execution_count": 448,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_10k = pd.read_csv('10k_Datasets/10k_april_tweets.csv', index_col=0)\n",
    "apr_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-04-01', '2020-04-02', '2020-04-03', nan, 'Thomas Paine ✊',\n",
       "       '2020-04-04', '2020-04-05', '2020-04-06', '2020-04-07',\n",
       "       '2020-04-08', '2020-04-09', '2020-04-10', '2020-04-11',\n",
       "       '2020-04-12', '2020-04-13', '2020-04-14', '2020-04-15',\n",
       "       '2020-04-16', '2020-04-17', '2020-04-18', '2020-04-19',\n",
       "       '2020-04-20', '2020-04-21', '2020-04-22', '2020-04-23',\n",
       "       '2020-04-24', '2020-04-25', '2020-04-26', '2020-04-27',\n",
       "       '2020-04-28', '2020-04-29', '2020-04-30'], dtype=object)"
      ]
     },
     "execution_count": 450,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_10k['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mar28_10k = pd.read_csv('Datasets/alt_search_march17_tweets.csv', index_col=0)\n",
    "mar28_10k.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "jan_clean = jan.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(487435, 17)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(476761, 17)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.dropna(how='all', inplace=True)\n",
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-01-01', '2020-01-02', '2020-01-03', '2020-01-04',\n",
       "       'Dream Doctor®', '2020-01-05', '2020-01-06', '2020-01-07',\n",
       "       '2020-01-08', 'We The Curators', '2020-01-09', '2020-01-10',\n",
       "       '2020-01-11', '2020-01-12', '2020-01-13', '2020-01-14',\n",
       "       '2020-01-15', '2020-01-16', '2020-01-17', '2020-01-18',\n",
       "       '2020-01-19', '2020-01-20', '2020-01-21', '2020-01-22',\n",
       "       '2020-01-23', '2020-01-24', '2020-01-25', '2020-01-26',\n",
       "       '2020-01-27', '2020-01-28', '2020-01-29', '2020-01-30',\n",
       "       '2020-01-31'], dtype=object)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>98304</th>\n",
       "      <td>[]</td>\n",
       "      <td>2.991984e+09</td>\n",
       "      <td>WeTheCurators2</td>\n",
       "      <td>We The Curators</td>\n",
       "      <td>https://twitter.com/WeTheCurators2/status/1214...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2991984139', 'username': 'WeTheC...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  conversation_id      created_at             date  \\\n",
       "98304  []     2.991984e+09  WeTheCurators2  We The Curators   \n",
       "\n",
       "                                                   tweet hashtags  user_id  \\\n",
       "98304  https://twitter.com/WeTheCurators2/status/1214...    False      0.0   \n",
       "\n",
       "      username name link retweet  \\\n",
       "98304        0    0  NaN     NaN   \n",
       "\n",
       "                                                  nlikes  nreplies  nretweets  \\\n",
       "98304  [{'user_id': '2991984139', 'username': 'WeTheC...       NaN        NaN   \n",
       "\n",
       "      quote_url  retweet_id reply_to  \n",
       "98304       NaN         NaN      NaN  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean[jan_clean['date'] == 'We The Curators']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(476759, 17)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.drop(index=[28907, 98304], inplace=True)\n",
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-01-01', '2020-01-02', '2020-01-03', '2020-01-04',\n",
       "       '2020-01-05', '2020-01-06', '2020-01-07', '2020-01-08',\n",
       "       '2020-01-09', '2020-01-10', '2020-01-11', '2020-01-12',\n",
       "       '2020-01-13', '2020-01-14', '2020-01-15', '2020-01-16',\n",
       "       '2020-01-17', '2020-01-18', '2020-01-19', '2020-01-20',\n",
       "       '2020-01-21', '2020-01-22', '2020-01-23', '2020-01-24',\n",
       "       '2020-01-25', '2020-01-26', '2020-01-27', '2020-01-28',\n",
       "       '2020-01-29', '2020-01-30', '2020-01-31'], dtype=object)"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(476759, 17)"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(450482, 17)"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean = jan_clean.drop_duplicates(subset='tweet')\n",
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "conversation_id         0\n",
       "created_at              0\n",
       "date                    0\n",
       "tweet                   0\n",
       "hashtags                2\n",
       "user_id                 2\n",
       "username                2\n",
       "name                   18\n",
       "link                    2\n",
       "retweet                 2\n",
       "nlikes                  2\n",
       "nreplies                2\n",
       "nretweets               2\n",
       "quote_url          410160\n",
       "retweet_id         450482\n",
       "reply_to                2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(450464, 17)"
      ]
     },
     "execution_count": 366,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.dropna(subset=['name'], inplace=True)\n",
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.577923e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>SpongeBob: slides down Sand Mountain\\nThe tree...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.099210e+18</td>\n",
       "      <td>Matt04181</td>\n",
       "      <td>Matt0417 (CEO of POOP)</td>\n",
       "      <td>https://twitter.com/Matt04181/status/121252388...</td>\n",
       "      <td>False</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1099209894295678977', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.577923e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Started 2020 with freshly washed bed sheets, a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>5.777754e+07</td>\n",
       "      <td>standtallx</td>\n",
       "      <td>Emma 🧛🏻‍♀️</td>\n",
       "      <td>https://twitter.com/standtallx/status/12125238...</td>\n",
       "      <td>False</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '57777541', 'username': 'standtal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.212421e+18</td>\n",
       "      <td>1.577923e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Using the coverage of his yellow card to mask ...</td>\n",
       "      <td>['#classicjose', '#coverup']</td>\n",
       "      <td>3.466098e+08</td>\n",
       "      <td>5_Times_LFC</td>\n",
       "      <td>6 Times LFC</td>\n",
       "      <td>https://twitter.com/5_Times_LFC/status/1212523...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '346609834', 'username': '5_Times...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.212403e+18</td>\n",
       "      <td>1.577923e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>The mask  pic.twitter.com/vHi9pC9S62</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.070006e+18</td>\n",
       "      <td>kagura_shelby</td>\n",
       "      <td>mugiwara no</td>\n",
       "      <td>https://twitter.com/kagura_shelby/status/12125...</td>\n",
       "      <td>False</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1070005555777945600', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.212524e+18</td>\n",
       "      <td>1.212513e+18</td>\n",
       "      <td>1.577923e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>I know what you mean, I've been depressed too ...</td>\n",
       "      <td>['#hugs']</td>\n",
       "      <td>4.865904e+07</td>\n",
       "      <td>IBdaSweet1</td>\n",
       "      <td>Luwamba L.J.Taylor</td>\n",
       "      <td>https://twitter.com/IBdaSweet1/status/12125237...</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '48659042', 'username': 'IBdaSwee...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  conversation_id    created_at        date  \\\n",
       "0  1.212524e+18     1.212524e+18  1.577923e+12  2020-01-01   \n",
       "1  1.212524e+18     1.212524e+18  1.577923e+12  2020-01-01   \n",
       "2  1.212524e+18     1.212421e+18  1.577923e+12  2020-01-01   \n",
       "3  1.212524e+18     1.212403e+18  1.577923e+12  2020-01-01   \n",
       "4  1.212524e+18     1.212513e+18  1.577923e+12  2020-01-01   \n",
       "\n",
       "                                               tweet  \\\n",
       "0  SpongeBob: slides down Sand Mountain\\nThe tree...   \n",
       "1  Started 2020 with freshly washed bed sheets, a...   \n",
       "2  Using the coverage of his yellow card to mask ...   \n",
       "3               The mask  pic.twitter.com/vHi9pC9S62   \n",
       "4  I know what you mean, I've been depressed too ...   \n",
       "\n",
       "                       hashtags       user_id       username  \\\n",
       "0                            []  1.099210e+18      Matt04181   \n",
       "1                            []  5.777754e+07     standtallx   \n",
       "2  ['#classicjose', '#coverup']  3.466098e+08    5_Times_LFC   \n",
       "3                            []  1.070006e+18  kagura_shelby   \n",
       "4                     ['#hugs']  4.865904e+07     IBdaSweet1   \n",
       "\n",
       "                     name                                               link  \\\n",
       "0  Matt0417 (CEO of POOP)  https://twitter.com/Matt04181/status/121252388...   \n",
       "1              Emma 🧛🏻‍♀️  https://twitter.com/standtallx/status/12125238...   \n",
       "2             6 Times LFC  https://twitter.com/5_Times_LFC/status/1212523...   \n",
       "3             mugiwara no  https://twitter.com/kagura_shelby/status/12125...   \n",
       "4      Luwamba L.J.Taylor  https://twitter.com/IBdaSweet1/status/12125237...   \n",
       "\n",
       "  retweet  nlikes  nreplies  nretweets quote_url  retweet_id  \\\n",
       "0   False     4.0       0.0        0.0       NaN         NaN   \n",
       "1   False     3.0       2.0        0.0       NaN         NaN   \n",
       "2   False     0.0       0.0        0.0       NaN         NaN   \n",
       "3   False     3.0       0.0        0.0       NaN         NaN   \n",
       "4   False     1.0       0.0        0.0       NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '1099209894295678977', 'username'...  \n",
       "1  [{'user_id': '57777541', 'username': 'standtal...  \n",
       "2  [{'user_id': '346609834', 'username': '5_Times...  \n",
       "3  [{'user_id': '1070005555777945600', 'username'...  \n",
       "4  [{'user_id': '48659042', 'username': 'IBdaSwee...  "
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.reset_index(drop=True, inplace=True)\n",
    "jan_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [],
   "source": [
    "jan_clean.to_csv('Datasets/jan_tweets_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(626691, 17)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean = feb.copy()\n",
    "feb_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(584161, 17)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean.dropna(how='all', inplace=True)\n",
    "feb_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-02-01', '2020-02-02', '2020-02-03', '2020-02-04',\n",
       "       '2020-02-05', '2020-02-06', '2020-02-07', '2020-02-08',\n",
       "       '2020-02-09', 'jack', '2020-02-10', '2020-02-11', '2020-02-12',\n",
       "       '2020-02-13', 'KennyWonKenOhBe', '2020-02-14', '2020-02-15',\n",
       "       '2020-02-16', '2020-02-17', 'AsiaNews EN', '2020-02-18', 'Femina',\n",
       "       '2020-02-19', '2020-02-20', '2020-02-21', '2020-02-22',\n",
       "       '2020-02-23', 'We The Curators', '2020-02-24', '2020-02-25',\n",
       "       '2020-02-26', '2020-02-27', '2020-02-28', 'Vivian Kwan',\n",
       "       '2020-02-29'], dtype=object)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>156900</th>\n",
       "      <td>['#end_of_the_world', '#china', '#wuhan', '#as...</td>\n",
       "      <td>1.071292e+18</td>\n",
       "      <td>jack24415701</td>\n",
       "      <td>jack</td>\n",
       "      <td>https://twitter.com/jack24415701/status/122646...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1071291991873413121', 'username'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       id  conversation_id  \\\n",
       "156900  ['#end_of_the_world', '#china', '#wuhan', '#as...     1.071292e+18   \n",
       "\n",
       "          created_at  date                                              tweet  \\\n",
       "156900  jack24415701  jack  https://twitter.com/jack24415701/status/122646...   \n",
       "\n",
       "       hashtags  user_id username name link retweet  \\\n",
       "156900    False      0.0        0    0  NaN     NaN   \n",
       "\n",
       "                                                   nlikes  nreplies  \\\n",
       "156900  [{'user_id': '1071291991873413121', 'username'...       NaN   \n",
       "\n",
       "        nretweets quote_url  retweet_id reply_to  \n",
       "156900        NaN       NaN         NaN      NaN  "
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean[feb_clean['date'] == 'jack']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(584155, 17)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean.drop(index=[156900], inplace=True)\n",
    "feb_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-02-01', '2020-02-02', '2020-02-03', '2020-02-04',\n",
       "       '2020-02-05', '2020-02-06', '2020-02-07', '2020-02-08',\n",
       "       '2020-02-09', '2020-02-10', '2020-02-11', '2020-02-12',\n",
       "       '2020-02-13', '2020-02-14', '2020-02-15', '2020-02-16',\n",
       "       '2020-02-17', '2020-02-18', '2020-02-19', '2020-02-20',\n",
       "       '2020-02-21', '2020-02-22', '2020-02-23', '2020-02-24',\n",
       "       '2020-02-25', '2020-02-26', '2020-02-27', '2020-02-28',\n",
       "       '2020-02-29'], dtype=object)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(536891, 17)"
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean = feb_clean.drop_duplicates(subset='tweet')\n",
    "feb_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "conversation_id         0\n",
       "created_at              0\n",
       "date                    0\n",
       "tweet                   0\n",
       "hashtags                6\n",
       "user_id                 6\n",
       "username                8\n",
       "name                   24\n",
       "link                    6\n",
       "retweet                 6\n",
       "nlikes                  6\n",
       "nreplies                6\n",
       "nretweets               6\n",
       "quote_url          491745\n",
       "retweet_id         536891\n",
       "reply_to                6\n",
       "dtype: int64"
      ]
     },
     "execution_count": 371,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(536865, 17)"
      ]
     },
     "execution_count": 374,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean.dropna(subset=['name', 'username'], inplace=True)\n",
    "feb_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.580602e+12</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>Bought Bruno Fernandes, play him at CDM? Wtf a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.221553e+09</td>\n",
       "      <td>chong_wh96</td>\n",
       "      <td>Chong_wH</td>\n",
       "      <td>https://twitter.com/chong_wh96/status/12237578...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2221552537', 'username': 'chong_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.580602e+12</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>My sister sent me $90 worth of face masks and ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.075768e+09</td>\n",
       "      <td>Yamanik30</td>\n",
       "      <td>ACAB</td>\n",
       "      <td>https://twitter.com/Yamanik30/status/122375787...</td>\n",
       "      <td>False</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3075767827', 'username': 'Yamani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.580602e+12</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>Because Nigeria is currently 51% Muslim and th...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.184187e+07</td>\n",
       "      <td>Dimplez</td>\n",
       "      <td>Dimplez 🇳🇬</td>\n",
       "      <td>https://twitter.com/Dimplez/status/12237578717...</td>\n",
       "      <td>False</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>https://twitter.com/JesseLehrich/status/122336...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '21841872', 'username': 'Dimplez'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.580602e+12</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>When you were a mask to stop the cronavirus bu...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.209989e+18</td>\n",
       "      <td>Harry31348302</td>\n",
       "      <td>Harry</td>\n",
       "      <td>https://twitter.com/Harry31348302/status/12237...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1209988745790918656', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1.223757e+18</td>\n",
       "      <td>1.580602e+12</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>This 'MeshGraphic' implementation supports:\\n\\...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.478940e+09</td>\n",
       "      <td>Softdrink_117</td>\n",
       "      <td>Softdrink 117</td>\n",
       "      <td>https://twitter.com/Softdrink_117/status/12237...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1478940204', 'username': 'Softdr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  conversation_id    created_at        date  \\\n",
       "0  1.223758e+18     1.223758e+18  1.580602e+12  2020-02-01   \n",
       "1  1.223758e+18     1.223758e+18  1.580602e+12  2020-02-01   \n",
       "2  1.223758e+18     1.223758e+18  1.580602e+12  2020-02-01   \n",
       "3  1.223758e+18     1.223758e+18  1.580602e+12  2020-02-01   \n",
       "4  1.223758e+18     1.223757e+18  1.580602e+12  2020-02-01   \n",
       "\n",
       "                                               tweet hashtags       user_id  \\\n",
       "0  Bought Bruno Fernandes, play him at CDM? Wtf a...       []  2.221553e+09   \n",
       "1  My sister sent me $90 worth of face masks and ...       []  3.075768e+09   \n",
       "2  Because Nigeria is currently 51% Muslim and th...       []  2.184187e+07   \n",
       "3  When you were a mask to stop the cronavirus bu...       []  1.209989e+18   \n",
       "4  This 'MeshGraphic' implementation supports:\\n\\...       []  1.478940e+09   \n",
       "\n",
       "        username           name  \\\n",
       "0     chong_wh96       Chong_wH   \n",
       "1      Yamanik30           ACAB   \n",
       "2        Dimplez     Dimplez 🇳🇬   \n",
       "3  Harry31348302          Harry   \n",
       "4  Softdrink_117  Softdrink 117   \n",
       "\n",
       "                                                link retweet  nlikes  \\\n",
       "0  https://twitter.com/chong_wh96/status/12237578...   False     0.0   \n",
       "1  https://twitter.com/Yamanik30/status/122375787...   False     6.0   \n",
       "2  https://twitter.com/Dimplez/status/12237578717...   False    16.0   \n",
       "3  https://twitter.com/Harry31348302/status/12237...   False     0.0   \n",
       "4  https://twitter.com/Softdrink_117/status/12237...   False     0.0   \n",
       "\n",
       "   nreplies  nretweets                                          quote_url  \\\n",
       "0       0.0        0.0                                                NaN   \n",
       "1       0.0        0.0                                                NaN   \n",
       "2       0.0        3.0  https://twitter.com/JesseLehrich/status/122336...   \n",
       "3       0.0        0.0                                                NaN   \n",
       "4       0.0        0.0                                                NaN   \n",
       "\n",
       "   retweet_id                                           reply_to  \n",
       "0         NaN  [{'user_id': '2221552537', 'username': 'chong_...  \n",
       "1         NaN  [{'user_id': '3075767827', 'username': 'Yamani...  \n",
       "2         NaN   [{'user_id': '21841872', 'username': 'Dimplez'}]  \n",
       "3         NaN  [{'user_id': '1209988745790918656', 'username'...  \n",
       "4         NaN  [{'user_id': '1478940204', 'username': 'Softdr...  "
      ]
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean.reset_index(drop=True, inplace=True)\n",
    "feb_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [],
   "source": [
    "feb_clean.to_csv('Datasets/feb_tweets_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(737002, 17)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean = mar01.copy()\n",
    "mar01_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(736996, 17)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean.dropna(how='all', inplace=True)\n",
    "mar01_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-03-01', 'Vivian Kwan', '2020-03-02', '2020-03-03',\n",
       "       'James Clarke', '2020-03-04', '2020-03-05', '2020-03-06',\n",
       "       '2020-03-07', '2020-03-08', '2020-03-09',\n",
       "       'Haloo Automation Equipment', 'Celecare Medical', '2020-03-10',\n",
       "       '2020-03-11', 'AsiaNews EN', 'Dianna Olukotun', 'Winn🔥',\n",
       "       '2020-03-12', '2020-03-13', '2020-03-14', '2020-03-15',\n",
       "       '2020-03-16'], dtype=object)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>414281</th>\n",
       "      <td>[]</td>\n",
       "      <td>1.204048e+18</td>\n",
       "      <td>SherwinCaliwag</td>\n",
       "      <td>Winn🔥</td>\n",
       "      <td>https://twitter.com/SherwinCaliwag/status/1237...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1204047724259078144', 'username'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  conversation_id      created_at   date  \\\n",
       "414281  []     1.204048e+18  SherwinCaliwag  Winn🔥   \n",
       "\n",
       "                                                    tweet hashtags  user_id  \\\n",
       "414281  https://twitter.com/SherwinCaliwag/status/1237...    False      0.0   \n",
       "\n",
       "       username name link retweet  \\\n",
       "414281        0    0  NaN     NaN   \n",
       "\n",
       "                                                   nlikes  nreplies  \\\n",
       "414281  [{'user_id': '1204047724259078144', 'username'...       NaN   \n",
       "\n",
       "        nretweets quote_url  retweet_id reply_to  \n",
       "414281        NaN       NaN         NaN      NaN  "
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean[mar01_clean['date'] == 'Winn🔥']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(736989, 17)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean.drop(index=[24153, 82771, 315351, 316618, 381513, 393868, 414281], inplace=True)\n",
    "mar01_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-03-01', '2020-03-02', '2020-03-03', '2020-03-04',\n",
       "       '2020-03-05', '2020-03-06', '2020-03-07', '2020-03-08',\n",
       "       '2020-03-09', '2020-03-10', '2020-03-11', '2020-03-12',\n",
       "       '2020-03-13', '2020-03-14', '2020-03-15', '2020-03-16'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(695994, 17)"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean = mar01_clean.drop_duplicates(subset='tweet')\n",
    "mar01_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "conversation_id         0\n",
       "created_at              0\n",
       "date                    0\n",
       "tweet                   0\n",
       "hashtags                7\n",
       "user_id                 7\n",
       "username                7\n",
       "name                   36\n",
       "link                    7\n",
       "retweet                 7\n",
       "nlikes                  7\n",
       "nreplies                7\n",
       "nretweets               7\n",
       "quote_url          640207\n",
       "retweet_id         695994\n",
       "reply_to                7\n",
       "dtype: int64"
      ]
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(695958, 17)"
      ]
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean.dropna(subset=['name', 'username'], inplace=True)\n",
    "mar01_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1234267158877818880</td>\n",
       "      <td>1.234266e+18</td>\n",
       "      <td>1583107196000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Wearing a mask on a train track somewhere in B...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.222977e+18</td>\n",
       "      <td>KirkP31369321</td>\n",
       "      <td>Kirks Poutine</td>\n",
       "      <td>https://twitter.com/KirkP31369321/status/12342...</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1222977020705091588', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1234267142259953664</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107192000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Were willing to start wearing surgical masks e...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.317655e+09</td>\n",
       "      <td>KCUNLEASHED96</td>\n",
       "      <td>KC UNLEASHED96</td>\n",
       "      <td>https://twitter.com/KCUNLEASHED96/status/12342...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>https://twitter.com/sssniperwolf/status/123426...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1317654776', 'username': 'KCUNLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1234267140955336704</td>\n",
       "      <td>1.234254e+18</td>\n",
       "      <td>1583107192000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>It's target are your lungs, don't give it acce...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.199808e+09</td>\n",
       "      <td>1spark780</td>\n",
       "      <td>Judy</td>\n",
       "      <td>https://twitter.com/1spark780/status/123426714...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3199808368', 'username': '1spark...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1234267136018665472</td>\n",
       "      <td>1.234259e+18</td>\n",
       "      <td>1583107191000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>The new police issue masks might not prevent c...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.931759e+09</td>\n",
       "      <td>hueman_being</td>\n",
       "      <td>Ken Parker</td>\n",
       "      <td>https://twitter.com/hueman_being/status/123426...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3931759154', 'username': 'hueman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1234267124304154625</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107188000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>hey i’m sad so i’m gonna do a face mask and th...</td>\n",
       "      <td>[]</td>\n",
       "      <td>4.252255e+08</td>\n",
       "      <td>beratings</td>\n",
       "      <td>baby bug</td>\n",
       "      <td>https://twitter.com/beratings/status/123426712...</td>\n",
       "      <td>False</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '425225495', 'username': 'beratin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  conversation_id     created_at        date  \\\n",
       "0  1234267158877818880     1.234266e+18  1583107196000  2020-03-01   \n",
       "1  1234267142259953664     1.234267e+18  1583107192000  2020-03-01   \n",
       "2  1234267140955336704     1.234254e+18  1583107192000  2020-03-01   \n",
       "3  1234267136018665472     1.234259e+18  1583107191000  2020-03-01   \n",
       "4  1234267124304154625     1.234267e+18  1583107188000  2020-03-01   \n",
       "\n",
       "                                               tweet hashtags       user_id  \\\n",
       "0  Wearing a mask on a train track somewhere in B...       []  1.222977e+18   \n",
       "1  Were willing to start wearing surgical masks e...       []  1.317655e+09   \n",
       "2  It's target are your lungs, don't give it acce...       []  3.199808e+09   \n",
       "3  The new police issue masks might not prevent c...       []  3.931759e+09   \n",
       "4  hey i’m sad so i’m gonna do a face mask and th...       []  4.252255e+08   \n",
       "\n",
       "        username            name  \\\n",
       "0  KirkP31369321   Kirks Poutine   \n",
       "1  KCUNLEASHED96  KC UNLEASHED96   \n",
       "2      1spark780            Judy   \n",
       "3   hueman_being      Ken Parker   \n",
       "4      beratings        baby bug   \n",
       "\n",
       "                                                link retweet  nlikes  \\\n",
       "0  https://twitter.com/KirkP31369321/status/12342...   False     1.0   \n",
       "1  https://twitter.com/KCUNLEASHED96/status/12342...   False     0.0   \n",
       "2  https://twitter.com/1spark780/status/123426714...   False     0.0   \n",
       "3  https://twitter.com/hueman_being/status/123426...   False     0.0   \n",
       "4  https://twitter.com/beratings/status/123426712...   False    14.0   \n",
       "\n",
       "   nreplies  nretweets                                          quote_url  \\\n",
       "0       0.0        0.0                                                NaN   \n",
       "1       0.0        0.0  https://twitter.com/sssniperwolf/status/123426...   \n",
       "2       0.0        0.0                                                NaN   \n",
       "3       0.0        0.0                                                NaN   \n",
       "4       1.0        0.0                                                NaN   \n",
       "\n",
       "   retweet_id                                           reply_to  \n",
       "0         NaN  [{'user_id': '1222977020705091588', 'username'...  \n",
       "1         NaN  [{'user_id': '1317654776', 'username': 'KCUNLE...  \n",
       "2         NaN  [{'user_id': '3199808368', 'username': '1spark...  \n",
       "3         NaN  [{'user_id': '3931759154', 'username': 'hueman...  \n",
       "4         NaN  [{'user_id': '425225495', 'username': 'beratin...  "
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean.reset_index(drop=True, inplace=True)\n",
    "mar01_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [],
   "source": [
    "mar01_clean.to_csv('Datasets/mar01_tweets_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(695958, 17)"
      ]
     },
     "execution_count": 402,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean = pd.read_csv('Datasets/mar01_tweets_clean.csv', index_col=0)\n",
    "mar01_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9734, 17)"
      ]
     },
     "execution_count": 403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_10k = pd.read_csv('10k_Datasets/10k_march17_tweets.csv', index_col=0)\n",
    "mar17_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(133560, 17)"
      ]
     },
     "execution_count": 404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar18_10k = pd.read_csv('10k_Datasets/10k_march18_tweets.csv', index_col=0, encoding='utf8', engine='python')\n",
    "mar18_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(143294, 17)"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_clean_10k = pd.concat([mar17_10k, mar18_10k])\n",
    "mar17_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(143294, 17)"
      ]
     },
     "execution_count": 406,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_clean_10k.reset_index(drop=True, inplace=True)\n",
    "mar17_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(143292, 17)"
      ]
     },
     "execution_count": 407,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_clean_10k.dropna(how='all', inplace=True)\n",
    "mar17_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-03-17', '2020-03-18', '2020-03-19', 'Combo Gadgets',\n",
       "       '2020-03-20', '2020-03-21', '2020-03-22', '2020-03-23',\n",
       "       '2020-03-24', '2020-03-25', '2020-03-26', '2020-03-27',\n",
       "       '2020-03-28', '2020-03-29', 'Stat Butler', '2020-03-30',\n",
       "       '2020-03-31'], dtype=object)"
      ]
     },
     "execution_count": 409,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_clean_10k['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>121954</th>\n",
       "      <td>[]</td>\n",
       "      <td>9.819696e+17</td>\n",
       "      <td>stat_butler</td>\n",
       "      <td>Stat Butler</td>\n",
       "      <td>https://twitter.com/stat_butler/status/1244265...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '981969608277135360', 'username':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122265</th>\n",
       "      <td>[]</td>\n",
       "      <td>9.819696e+17</td>\n",
       "      <td>stat_butler</td>\n",
       "      <td>Stat Butler</td>\n",
       "      <td>https://twitter.com/stat_butler/status/1244260...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '981969608277135360', 'username':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  conversation_id   created_at         date  \\\n",
       "121954  []     9.819696e+17  stat_butler  Stat Butler   \n",
       "122265  []     9.819696e+17  stat_butler  Stat Butler   \n",
       "\n",
       "                                                    tweet hashtags  user_id  \\\n",
       "121954  https://twitter.com/stat_butler/status/1244265...    False      0.0   \n",
       "122265  https://twitter.com/stat_butler/status/1244260...    False      0.0   \n",
       "\n",
       "       username name link retweet  \\\n",
       "121954        0    0  NaN     NaN   \n",
       "122265        0    1  NaN     NaN   \n",
       "\n",
       "                                                   nlikes  nreplies  \\\n",
       "121954  [{'user_id': '981969608277135360', 'username':...       NaN   \n",
       "122265  [{'user_id': '981969608277135360', 'username':...       NaN   \n",
       "\n",
       "        nretweets quote_url  retweet_id reply_to  \n",
       "121954        NaN      None         NaN     None  \n",
       "122265        NaN      None         NaN     None  "
      ]
     },
     "execution_count": 410,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_clean_10k[mar17_clean_10k['date'] == 'Stat Butler']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(143289, 17)"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_clean_10k.drop(index=[22752, 121954, 122265], inplace=True)\n",
    "mar17_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(143289, 17)"
      ]
     },
     "execution_count": 412,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar17_clean_10k.reset_index(drop=True, inplace=True)\n",
    "mar17_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(839247, 17)"
      ]
     },
     "execution_count": 418,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean_10k_from_17th = pd.concat([mar01_clean, mar17_clean_10k])\n",
    "mar_clean_10k_from_17th.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(830115, 17)"
      ]
     },
     "execution_count": 431,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean_10k_from_17th = mar_clean_10k_from_17th.drop_duplicates(subset='tweet')\n",
    "mar_clean_10k_from_17th.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "conversation_id         0\n",
       "created_at              0\n",
       "date                    0\n",
       "tweet                   0\n",
       "hashtags               10\n",
       "user_id                10\n",
       "username               10\n",
       "name                   42\n",
       "link                   10\n",
       "retweet                10\n",
       "nlikes                 10\n",
       "nreplies               10\n",
       "nretweets              10\n",
       "quote_url          764081\n",
       "retweet_id         830115\n",
       "reply_to               10\n",
       "dtype: int64"
      ]
     },
     "execution_count": 432,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean_10k_from_17th.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eric/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(830073, 17)"
      ]
     },
     "execution_count": 433,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean_10k_from_17th.dropna(subset=['name', 'username'], inplace=True)\n",
    "mar_clean_10k_from_17th.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(830073, 17)"
      ]
     },
     "execution_count": 436,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_clean_10k_from_17th.reset_index(drop=True, inplace=True)\n",
    "mar_clean_10k_from_17th.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "metadata": {},
   "outputs": [],
   "source": [
    "mar_clean_10k_from_17th.to_csv('Datasets/mar_tweets_clean_10k_from_17th.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(334598, 17)"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean_10k = apr_10k.copy()\n",
    "apr_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(299811, 17)"
      ]
     },
     "execution_count": 452,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean_10k.dropna(how='all', inplace=True)\n",
    "apr_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "conversation_id         0\n",
       "created_at              0\n",
       "date                    0\n",
       "tweet                   0\n",
       "hashtags             2019\n",
       "user_id              2019\n",
       "username             2019\n",
       "name                 2025\n",
       "link                 2020\n",
       "retweet              2020\n",
       "nlikes               2019\n",
       "nreplies             2020\n",
       "nretweets            2020\n",
       "quote_url          280944\n",
       "retweet_id         299811\n",
       "reply_to             2020\n",
       "dtype: int64"
      ]
     },
     "execution_count": 455,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean_10k.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(297785, 17)"
      ]
     },
     "execution_count": 458,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean_10k.dropna(subset=['name', 'username', 'link'], inplace=True)\n",
    "apr_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2020-04-01', '2020-04-02', '2020-04-03', '2020-04-04',\n",
       "       '2020-04-05', '2020-04-06', '2020-04-07', '2020-04-08',\n",
       "       '2020-04-09', '2020-04-10', '2020-04-11', '2020-04-12',\n",
       "       '2020-04-13', '2020-04-14', '2020-04-15', '2020-04-16',\n",
       "       '2020-04-17', '2020-04-18', '2020-04-19', '2020-04-20',\n",
       "       '2020-04-21', '2020-04-22', '2020-04-23', '2020-04-24',\n",
       "       '2020-04-25', '2020-04-26', '2020-04-27', '2020-04-28',\n",
       "       '2020-04-29', '2020-04-30'], dtype=object)"
      ]
     },
     "execution_count": 460,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean_10k['date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(297785, 17)"
      ]
     },
     "execution_count": 461,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_clean_10k.reset_index(drop=True, inplace=True)\n",
    "apr_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {},
   "outputs": [],
   "source": [
    "apr_clean_10k.to_csv('Datasets/apr_tweets_clean_10k.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(476759, 17)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1212523477896294400</td>\n",
       "      <td>1.212523e+18</td>\n",
       "      <td>1577923099000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>https://mltshp.com/p/1HLSB  \"This is the deat...</td>\n",
       "      <td>[]</td>\n",
       "      <td>8.431951e+17</td>\n",
       "      <td>Best_of_MLTSHP</td>\n",
       "      <td>MLTSHP</td>\n",
       "      <td>https://twitter.com/Best_of_MLTSHP/status/1212...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '843195118924546048', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>1212518043353530368</td>\n",
       "      <td>1.212518e+18</td>\n",
       "      <td>1577921803000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Ready to heard this? Homemade hair treatment a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.044739e+18</td>\n",
       "      <td>lumora_lu</td>\n",
       "      <td>MiLu</td>\n",
       "      <td>https://twitter.com/lumora_lu/status/121251804...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1044738746913034241', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>1212517557690748928</td>\n",
       "      <td>1.212518e+18</td>\n",
       "      <td>1577921687000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>We all know what a joke #CCPChina is when it c...</td>\n",
       "      <td>['#ccpchina', '#sars', '#hk', '#hkers']</td>\n",
       "      <td>9.630325e+17</td>\n",
       "      <td>odiecher</td>\n",
       "      <td>odiecher</td>\n",
       "      <td>https://twitter.com/odiecher/status/1212517557...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://twitter.com/WBYeats1865/status/1212430...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '963032516759207936', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>1212515585562726401</td>\n",
       "      <td>1.200523e+18</td>\n",
       "      <td>1577921217000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Thought death masks gave you the horn freak</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.819219e+09</td>\n",
       "      <td>Hevysmoker</td>\n",
       "      <td>Ashley Collins</td>\n",
       "      <td>https://twitter.com/Hevysmoker/status/12125155...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2819219369', 'username': 'Hevysm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>1212510409858670593</td>\n",
       "      <td>1.212391e+18</td>\n",
       "      <td>1577919983000</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Interesting cult of death mask he’s wearing, 🤔...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.361956e+08</td>\n",
       "      <td>FaithR8s</td>\n",
       "      <td>❌Patriot Fan🇺🇸🗽☕️🎶❌</td>\n",
       "      <td>https://twitter.com/FaithR8s/status/1212510409...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '236195607', 'username': 'FaithR8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476736</th>\n",
       "      <td>1223108827530153985</td>\n",
       "      <td>1.223109e+18</td>\n",
       "      <td>1580446843000</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>TWICE will be continuing with their meet &amp; gre...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.246642e+09</td>\n",
       "      <td>misayeon</td>\n",
       "      <td>misa •ᴗ•</td>\n",
       "      <td>https://twitter.com/misayeon/status/1223108827...</td>\n",
       "      <td>False</td>\n",
       "      <td>5651</td>\n",
       "      <td>86.0</td>\n",
       "      <td>1343.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2246641690', 'username': 'misaye...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476743</th>\n",
       "      <td>1223108743304364033</td>\n",
       "      <td>1.223109e+18</td>\n",
       "      <td>1580446823000</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>“Doctor's advice about coronavirus:\\n\\n1. It i...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.854207e+09</td>\n",
       "      <td>AlwaysDesTitres</td>\n",
       "      <td>🧙🏻‍♂️🌱&amp; JDs💜</td>\n",
       "      <td>https://twitter.com/AlwaysDesTitres/status/122...</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1854206910', 'username': 'Always...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476745</th>\n",
       "      <td>1223108727164653570</td>\n",
       "      <td>1.223058e+18</td>\n",
       "      <td>1580446819000</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>The face mask is running out of stock in Hong ...</td>\n",
       "      <td>['#coronavirusoutbreak', '#hongkongprotests']</td>\n",
       "      <td>1.193545e+18</td>\n",
       "      <td>tree_louis</td>\n",
       "      <td>Louis K S Tang</td>\n",
       "      <td>https://twitter.com/tree_louis/status/12231087...</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1193544878703009792', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476750</th>\n",
       "      <td>1223108704867774466</td>\n",
       "      <td>1.223109e+18</td>\n",
       "      <td>1580446814000</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>Pro-#CCPChina thugs attacked kids in #YuenLong...</td>\n",
       "      <td>['#ccpchina', '#yuenlong', '#coronavirus', '#h...</td>\n",
       "      <td>8.041465e+08</td>\n",
       "      <td>AngusChTrix</td>\n",
       "      <td>angusCH</td>\n",
       "      <td>https://twitter.com/AngusChTrix/status/1223108...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://twitter.com/lukedepulford/status/12229...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '804146479', 'username': 'AngusCh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476756</th>\n",
       "      <td>1223108655731503105</td>\n",
       "      <td>1.223109e+18</td>\n",
       "      <td>1580446802000</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>To help fight the outbreak of pneumonia caused...</td>\n",
       "      <td>['#coronavirus']</td>\n",
       "      <td>8.777542e+07</td>\n",
       "      <td>ChinaDaily</td>\n",
       "      <td>China Daily</td>\n",
       "      <td>https://twitter.com/ChinaDaily/status/12231086...</td>\n",
       "      <td>False</td>\n",
       "      <td>64</td>\n",
       "      <td>3.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '87775422', 'username': 'ChinaDai...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>48155 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id  conversation_id     created_at        date  \\\n",
       "15      1212523477896294400     1.212523e+18  1577923099000  2020-01-01   \n",
       "201     1212518043353530368     1.212518e+18  1577921803000  2020-01-01   \n",
       "222     1212517557690748928     1.212518e+18  1577921687000  2020-01-01   \n",
       "327     1212515585562726401     1.200523e+18  1577921217000  2020-01-01   \n",
       "522     1212510409858670593     1.212391e+18  1577919983000  2020-01-01   \n",
       "...                     ...              ...            ...         ...   \n",
       "476736  1223108827530153985     1.223109e+18  1580446843000  2020-01-31   \n",
       "476743  1223108743304364033     1.223109e+18  1580446823000  2020-01-31   \n",
       "476745  1223108727164653570     1.223058e+18  1580446819000  2020-01-31   \n",
       "476750  1223108704867774466     1.223109e+18  1580446814000  2020-01-31   \n",
       "476756  1223108655731503105     1.223109e+18  1580446802000  2020-01-31   \n",
       "\n",
       "                                                    tweet  \\\n",
       "15       https://mltshp.com/p/1HLSB  \"This is the deat...   \n",
       "201     Ready to heard this? Homemade hair treatment a...   \n",
       "222     We all know what a joke #CCPChina is when it c...   \n",
       "327           Thought death masks gave you the horn freak   \n",
       "522     Interesting cult of death mask he’s wearing, 🤔...   \n",
       "...                                                   ...   \n",
       "476736  TWICE will be continuing with their meet & gre...   \n",
       "476743  “Doctor's advice about coronavirus:\\n\\n1. It i...   \n",
       "476745  The face mask is running out of stock in Hong ...   \n",
       "476750  Pro-#CCPChina thugs attacked kids in #YuenLong...   \n",
       "476756  To help fight the outbreak of pneumonia caused...   \n",
       "\n",
       "                                                 hashtags       user_id  \\\n",
       "15                                                     []  8.431951e+17   \n",
       "201                                                    []  1.044739e+18   \n",
       "222               ['#ccpchina', '#sars', '#hk', '#hkers']  9.630325e+17   \n",
       "327                                                    []  2.819219e+09   \n",
       "522                                                    []  2.361956e+08   \n",
       "...                                                   ...           ...   \n",
       "476736                                                 []  2.246642e+09   \n",
       "476743                                                 []  1.854207e+09   \n",
       "476745      ['#coronavirusoutbreak', '#hongkongprotests']  1.193545e+18   \n",
       "476750  ['#ccpchina', '#yuenlong', '#coronavirus', '#h...  8.041465e+08   \n",
       "476756                                   ['#coronavirus']  8.777542e+07   \n",
       "\n",
       "               username                 name  \\\n",
       "15       Best_of_MLTSHP               MLTSHP   \n",
       "201           lumora_lu                 MiLu   \n",
       "222            odiecher             odiecher   \n",
       "327          Hevysmoker       Ashley Collins   \n",
       "522            FaithR8s  ❌Patriot Fan🇺🇸🗽☕️🎶❌   \n",
       "...                 ...                  ...   \n",
       "476736         misayeon             misa •ᴗ•   \n",
       "476743  AlwaysDesTitres         🧙🏻‍♂️🌱& JDs💜   \n",
       "476745       tree_louis       Louis K S Tang   \n",
       "476750      AngusChTrix              angusCH   \n",
       "476756       ChinaDaily          China Daily   \n",
       "\n",
       "                                                     link retweet nlikes  \\\n",
       "15      https://twitter.com/Best_of_MLTSHP/status/1212...   False      1   \n",
       "201     https://twitter.com/lumora_lu/status/121251804...   False      0   \n",
       "222     https://twitter.com/odiecher/status/1212517557...   False      1   \n",
       "327     https://twitter.com/Hevysmoker/status/12125155...   False      0   \n",
       "522     https://twitter.com/FaithR8s/status/1212510409...   False      1   \n",
       "...                                                   ...     ...    ...   \n",
       "476736  https://twitter.com/misayeon/status/1223108827...   False   5651   \n",
       "476743  https://twitter.com/AlwaysDesTitres/status/122...   False      9   \n",
       "476745  https://twitter.com/tree_louis/status/12231087...   False      2   \n",
       "476750  https://twitter.com/AngusChTrix/status/1223108...   False      1   \n",
       "476756  https://twitter.com/ChinaDaily/status/12231086...   False     64   \n",
       "\n",
       "        nreplies  nretweets  \\\n",
       "15           0.0        1.0   \n",
       "201          0.0        0.0   \n",
       "222          1.0        1.0   \n",
       "327          0.0        0.0   \n",
       "522          0.0        1.0   \n",
       "...          ...        ...   \n",
       "476736      86.0     1343.0   \n",
       "476743       1.0        6.0   \n",
       "476745       0.0        1.0   \n",
       "476750       0.0        1.0   \n",
       "476756       3.0       17.0   \n",
       "\n",
       "                                                quote_url  retweet_id  \\\n",
       "15                                                    NaN         NaN   \n",
       "201                                                   NaN         NaN   \n",
       "222     https://twitter.com/WBYeats1865/status/1212430...         NaN   \n",
       "327                                                   NaN         NaN   \n",
       "522                                                   NaN         NaN   \n",
       "...                                                   ...         ...   \n",
       "476736                                                NaN         NaN   \n",
       "476743                                                NaN         NaN   \n",
       "476745                                                NaN         NaN   \n",
       "476750  https://twitter.com/lukedepulford/status/12229...         NaN   \n",
       "476756                                                NaN         NaN   \n",
       "\n",
       "                                                 reply_to  \n",
       "15      [{'user_id': '843195118924546048', 'username':...  \n",
       "201     [{'user_id': '1044738746913034241', 'username'...  \n",
       "222     [{'user_id': '963032516759207936', 'username':...  \n",
       "327     [{'user_id': '2819219369', 'username': 'Hevysm...  \n",
       "522     [{'user_id': '236195607', 'username': 'FaithR8...  \n",
       "...                                                   ...  \n",
       "476736  [{'user_id': '2246641690', 'username': 'misaye...  \n",
       "476743  [{'user_id': '1854206910', 'username': 'Always...  \n",
       "476745  [{'user_id': '1193544878703009792', 'username'...  \n",
       "476750  [{'user_id': '804146479', 'username': 'AngusCh...  \n",
       "476756  [{'user_id': '87775422', 'username': 'ChinaDai...  \n",
       "\n",
       "[48155 rows x 17 columns]"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filt = 'covid|dead|death|doctor|infect|novel|nurse|outbreak|rona|sars|viral|virus|wuhan'\n",
    "jan_clean[jan_clean['tweet'].str.contains(filt)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1223757879305367552</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601589000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>Bought Bruno Fernandes, play him at CDM? Wtf a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.221553e+09</td>\n",
       "      <td>chong_wh96</td>\n",
       "      <td>Chong_wH</td>\n",
       "      <td>https://twitter.com/chong_wh96/status/12237578...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2221552537', 'username': 'chong_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1223757878827159552</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601589000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>My sister sent me $90 worth of face masks and ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.075768e+09</td>\n",
       "      <td>Yamanik30</td>\n",
       "      <td>ACAB</td>\n",
       "      <td>https://twitter.com/Yamanik30/status/122375787...</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3075767827', 'username': 'Yamani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1223757866911326209</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601586000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>When you were a mask to stop the cronavirus bu...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.209989e+18</td>\n",
       "      <td>Harry31348302</td>\n",
       "      <td>Harry</td>\n",
       "      <td>https://twitter.com/Harry31348302/status/12237...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1209988745790918656', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1223757792244191232</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601568000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>this girl is deadass bragging about \"wearing m...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.156025e+09</td>\n",
       "      <td>JaceLucena</td>\n",
       "      <td>katka</td>\n",
       "      <td>https://twitter.com/JaceLucena/status/12237577...</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3156025472', 'username': 'JaceLu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1223757736166318082</td>\n",
       "      <td>1.223758e+18</td>\n",
       "      <td>1580601555000</td>\n",
       "      <td>2020-02-01</td>\n",
       "      <td>Sister is determined to infect me. Emerged fro...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.583843e+07</td>\n",
       "      <td>nyman11209</td>\n",
       "      <td>Hobonator</td>\n",
       "      <td>https://twitter.com/nyman11209/status/12237577...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '15838427', 'username': 'nyman112...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>584144</th>\n",
       "      <td>1233617929327063040</td>\n",
       "      <td>1.233618e+18</td>\n",
       "      <td>1582952408000</td>\n",
       "      <td>2020-02-29</td>\n",
       "      <td>Fear of the spreading coronavirus has led to a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.483876e+07</td>\n",
       "      <td>heralddispatch</td>\n",
       "      <td>The Herald-Dispatch</td>\n",
       "      <td>https://twitter.com/heralddispatch/status/1233...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '14838763', 'username': 'heralddi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>584146</th>\n",
       "      <td>1233617923480133633</td>\n",
       "      <td>1.233617e+18</td>\n",
       "      <td>1582952407000</td>\n",
       "      <td>2020-02-29</td>\n",
       "      <td>I was at a medical facility a couple year back...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.102966e+18</td>\n",
       "      <td>wildgooseLIVE</td>\n",
       "      <td>wildgoose</td>\n",
       "      <td>https://twitter.com/wildgooseLIVE/status/12336...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1102965924020264962', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>584149</th>\n",
       "      <td>1233617915003404288</td>\n",
       "      <td>1.233618e+18</td>\n",
       "      <td>1582952405000</td>\n",
       "      <td>2020-02-29</td>\n",
       "      <td>10 Best N95 Respirator Masks For Coronavirus  ...</td>\n",
       "      <td>['#top10', '#10bestn95respiratormasksforcorona...</td>\n",
       "      <td>1.156277e+09</td>\n",
       "      <td>WonderfulEngr</td>\n",
       "      <td>WonderfulEngineering</td>\n",
       "      <td>https://twitter.com/WonderfulEngr/status/12336...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1156277311', 'username': 'Wonder...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>584151</th>\n",
       "      <td>1233617899782201349</td>\n",
       "      <td>1.233618e+18</td>\n",
       "      <td>1582952401000</td>\n",
       "      <td>2020-02-29</td>\n",
       "      <td>Grateful to be back home for a while after thr...</td>\n",
       "      <td>['#coronavirus']</td>\n",
       "      <td>1.022851e+07</td>\n",
       "      <td>slhzdh</td>\n",
       "      <td>Kian Salehizadeh</td>\n",
       "      <td>https://twitter.com/slhzdh/status/123361789978...</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '10228512', 'username': 'slhzdh'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>584154</th>\n",
       "      <td>1233617894375735297</td>\n",
       "      <td>1.233618e+18</td>\n",
       "      <td>1582952400000</td>\n",
       "      <td>2020-02-29</td>\n",
       "      <td>HAPPENING NOW:  Our @A_AbdoolKarim is live on ...</td>\n",
       "      <td>['#coronavirusoutbreak', '#covid19']</td>\n",
       "      <td>1.288181e+09</td>\n",
       "      <td>Bhekisisa_MG</td>\n",
       "      <td>Bhekisisa</td>\n",
       "      <td>https://twitter.com/Bhekisisa_MG/status/123361...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1288180873', 'username': 'Bhekis...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>104519 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id  conversation_id     created_at        date  \\\n",
       "0       1223757879305367552     1.223758e+18  1580601589000  2020-02-01   \n",
       "1       1223757878827159552     1.223758e+18  1580601589000  2020-02-01   \n",
       "3       1223757866911326209     1.223758e+18  1580601586000  2020-02-01   \n",
       "11      1223757792244191232     1.223758e+18  1580601568000  2020-02-01   \n",
       "22      1223757736166318082     1.223758e+18  1580601555000  2020-02-01   \n",
       "...                     ...              ...            ...         ...   \n",
       "584144  1233617929327063040     1.233618e+18  1582952408000  2020-02-29   \n",
       "584146  1233617923480133633     1.233617e+18  1582952407000  2020-02-29   \n",
       "584149  1233617915003404288     1.233618e+18  1582952405000  2020-02-29   \n",
       "584151  1233617899782201349     1.233618e+18  1582952401000  2020-02-29   \n",
       "584154  1233617894375735297     1.233618e+18  1582952400000  2020-02-29   \n",
       "\n",
       "                                                    tweet  \\\n",
       "0       Bought Bruno Fernandes, play him at CDM? Wtf a...   \n",
       "1       My sister sent me $90 worth of face masks and ...   \n",
       "3       When you were a mask to stop the cronavirus bu...   \n",
       "11      this girl is deadass bragging about \"wearing m...   \n",
       "22      Sister is determined to infect me. Emerged fro...   \n",
       "...                                                   ...   \n",
       "584144  Fear of the spreading coronavirus has led to a...   \n",
       "584146  I was at a medical facility a couple year back...   \n",
       "584149  10 Best N95 Respirator Masks For Coronavirus  ...   \n",
       "584151  Grateful to be back home for a while after thr...   \n",
       "584154  HAPPENING NOW:  Our @A_AbdoolKarim is live on ...   \n",
       "\n",
       "                                                 hashtags       user_id  \\\n",
       "0                                                      []  2.221553e+09   \n",
       "1                                                      []  3.075768e+09   \n",
       "3                                                      []  1.209989e+18   \n",
       "11                                                     []  3.156025e+09   \n",
       "22                                                     []  1.583843e+07   \n",
       "...                                                   ...           ...   \n",
       "584144                                                 []  1.483876e+07   \n",
       "584146                                                 []  1.102966e+18   \n",
       "584149  ['#top10', '#10bestn95respiratormasksforcorona...  1.156277e+09   \n",
       "584151                                   ['#coronavirus']  1.022851e+07   \n",
       "584154               ['#coronavirusoutbreak', '#covid19']  1.288181e+09   \n",
       "\n",
       "              username                  name  \\\n",
       "0           chong_wh96              Chong_wH   \n",
       "1            Yamanik30                  ACAB   \n",
       "3        Harry31348302                 Harry   \n",
       "11          JaceLucena                 katka   \n",
       "22          nyman11209             Hobonator   \n",
       "...                ...                   ...   \n",
       "584144  heralddispatch   The Herald-Dispatch   \n",
       "584146   wildgooseLIVE             wildgoose   \n",
       "584149   WonderfulEngr  WonderfulEngineering   \n",
       "584151          slhzdh      Kian Salehizadeh   \n",
       "584154    Bhekisisa_MG             Bhekisisa   \n",
       "\n",
       "                                                     link retweet nlikes  \\\n",
       "0       https://twitter.com/chong_wh96/status/12237578...   False      0   \n",
       "1       https://twitter.com/Yamanik30/status/122375787...   False      6   \n",
       "3       https://twitter.com/Harry31348302/status/12237...   False      0   \n",
       "11      https://twitter.com/JaceLucena/status/12237577...   False      9   \n",
       "22      https://twitter.com/nyman11209/status/12237577...   False      1   \n",
       "...                                                   ...     ...    ...   \n",
       "584144  https://twitter.com/heralddispatch/status/1233...   False      0   \n",
       "584146  https://twitter.com/wildgooseLIVE/status/12336...   False      0   \n",
       "584149  https://twitter.com/WonderfulEngr/status/12336...   False      0   \n",
       "584151  https://twitter.com/slhzdh/status/123361789978...   False      8   \n",
       "584154  https://twitter.com/Bhekisisa_MG/status/123361...   False      1   \n",
       "\n",
       "        nreplies  nretweets quote_url  retweet_id  \\\n",
       "0            0.0        0.0       NaN         NaN   \n",
       "1            0.0        0.0       NaN         NaN   \n",
       "3            0.0        0.0       NaN         NaN   \n",
       "11           0.0        1.0       NaN         NaN   \n",
       "22           0.0        0.0       NaN         NaN   \n",
       "...          ...        ...       ...         ...   \n",
       "584144       0.0        0.0       NaN         NaN   \n",
       "584146       0.0        0.0       NaN         NaN   \n",
       "584149       0.0        0.0       NaN         NaN   \n",
       "584151       0.0        0.0       NaN         NaN   \n",
       "584154       0.0        0.0       NaN         NaN   \n",
       "\n",
       "                                                 reply_to  \n",
       "0       [{'user_id': '2221552537', 'username': 'chong_...  \n",
       "1       [{'user_id': '3075767827', 'username': 'Yamani...  \n",
       "3       [{'user_id': '1209988745790918656', 'username'...  \n",
       "11      [{'user_id': '3156025472', 'username': 'JaceLu...  \n",
       "22      [{'user_id': '15838427', 'username': 'nyman112...  \n",
       "...                                                   ...  \n",
       "584144  [{'user_id': '14838763', 'username': 'heralddi...  \n",
       "584146  [{'user_id': '1102965924020264962', 'username'...  \n",
       "584149  [{'user_id': '1156277311', 'username': 'Wonder...  \n",
       "584151    [{'user_id': '10228512', 'username': 'slhzdh'}]  \n",
       "584154  [{'user_id': '1288180873', 'username': 'Bhekis...  \n",
       "\n",
       "[104519 rows x 17 columns]"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filt = 'covid|dead|death|doctor|infect|novel|nurse|outbreak|rona|sars|viral|virus|wuhan'\n",
    "feb_clean[feb_clean['tweet'].str.contains(filt)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1234267136018665472</td>\n",
       "      <td>1.234259e+18</td>\n",
       "      <td>1583107191000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>The new police issue masks might not prevent c...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.931759e+09</td>\n",
       "      <td>hueman_being</td>\n",
       "      <td>Ken Parker</td>\n",
       "      <td>https://twitter.com/hueman_being/status/123426...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3931759154', 'username': 'hueman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1234267074568097792</td>\n",
       "      <td>1.233863e+18</td>\n",
       "      <td>1583107176000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Those *who are with the virus* should wear a m...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.087783e+07</td>\n",
       "      <td>danimadu</td>\n",
       "      <td>manequim da guarita da PF</td>\n",
       "      <td>https://twitter.com/danimadu/status/1234267074...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '20877834', 'username': 'danimadu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1234267018238611461</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107163000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>… I guess Nancy can put in masks for everyone ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.846698e+09</td>\n",
       "      <td>We4EverOvercome</td>\n",
       "      <td>Not Worthy Of Anything Remarkable</td>\n",
       "      <td>https://twitter.com/We4EverOvercome/status/123...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2846698472', 'username': 'We4Eve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1234266976853405697</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107153000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Washing your hands and disinfecting your elect...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.756134e+09</td>\n",
       "      <td>monicam_mendez</td>\n",
       "      <td>𝑴𝒐𝒏𝒊𝒄𝒂 𝑴𝒆𝒏𝒅𝒆𝒛 🌙</td>\n",
       "      <td>https://twitter.com/monicam_mendez/status/1234...</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://twitter.com/Pog_llins/status/123386973...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2756133519', 'username': 'monica...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1234266921521934336</td>\n",
       "      <td>1.234267e+18</td>\n",
       "      <td>1583107140000</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>Americans worried about coronavirus don't need...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.161490e+09</td>\n",
       "      <td>gtj654</td>\n",
       "      <td>Guy Johnson</td>\n",
       "      <td>https://twitter.com/gtj654/status/123426692152...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3161490206', 'username': 'gtj654'}]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736981</th>\n",
       "      <td>1239401044175540224</td>\n",
       "      <td>1.239401e+18</td>\n",
       "      <td>1584331210000</td>\n",
       "      <td>2020-03-16</td>\n",
       "      <td>corona virus finally giving me a reason to dro...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.589109e+09</td>\n",
       "      <td>jonwadec</td>\n",
       "      <td>jon</td>\n",
       "      <td>https://twitter.com/jonwadec/status/1239401044...</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2589108997', 'username': 'jonwad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736984</th>\n",
       "      <td>1239401022486982656</td>\n",
       "      <td>1.239401e+18</td>\n",
       "      <td>1584331205000</td>\n",
       "      <td>2020-03-16</td>\n",
       "      <td>2 questions: \\n\\nIs this a good time to finish...</td>\n",
       "      <td>[]</td>\n",
       "      <td>4.880158e+09</td>\n",
       "      <td>70bpmhiccups</td>\n",
       "      <td>Joe Seatter</td>\n",
       "      <td>https://twitter.com/70bpmhiccups/status/123940...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '4880157513', 'username': '70bpmh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736985</th>\n",
       "      <td>1239401020725309440</td>\n",
       "      <td>1.239347e+18</td>\n",
       "      <td>1584331204000</td>\n",
       "      <td>2020-03-16</td>\n",
       "      <td>The CDC says masks don't work. The virus is to...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.377558e+09</td>\n",
       "      <td>GunSensePA</td>\n",
       "      <td>Gun Sense PA</td>\n",
       "      <td>https://twitter.com/GunSensePA/status/12394010...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3377557565', 'username': 'GunSen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736986</th>\n",
       "      <td>1239401009228607488</td>\n",
       "      <td>1.239401e+18</td>\n",
       "      <td>1584331202000</td>\n",
       "      <td>2020-03-16</td>\n",
       "      <td>Those who have hoarded alcohol, masks, disinfe...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.664186e+08</td>\n",
       "      <td>maricapz</td>\n",
       "      <td>Marie Capanzana</td>\n",
       "      <td>https://twitter.com/maricapz/status/1239401009...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '266418579', 'username': 'maricap...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736987</th>\n",
       "      <td>1239401007572029440</td>\n",
       "      <td>1.239401e+18</td>\n",
       "      <td>1584331201000</td>\n",
       "      <td>2020-03-16</td>\n",
       "      <td>Doctor Oz said people are wearing surgical mas...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.543078e+07</td>\n",
       "      <td>jasdagod_</td>\n",
       "      <td>JAS DA GOD</td>\n",
       "      <td>https://twitter.com/jasdagod_/status/123940100...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '25430780', 'username': 'jasdagod...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>188691 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id  conversation_id     created_at        date  \\\n",
       "3       1234267136018665472     1.234259e+18  1583107191000  2020-03-01   \n",
       "11      1234267074568097792     1.233863e+18  1583107176000  2020-03-01   \n",
       "15      1234267018238611461     1.234267e+18  1583107163000  2020-03-01   \n",
       "23      1234266976853405697     1.234267e+18  1583107153000  2020-03-01   \n",
       "27      1234266921521934336     1.234267e+18  1583107140000  2020-03-01   \n",
       "...                     ...              ...            ...         ...   \n",
       "736981  1239401044175540224     1.239401e+18  1584331210000  2020-03-16   \n",
       "736984  1239401022486982656     1.239401e+18  1584331205000  2020-03-16   \n",
       "736985  1239401020725309440     1.239347e+18  1584331204000  2020-03-16   \n",
       "736986  1239401009228607488     1.239401e+18  1584331202000  2020-03-16   \n",
       "736987  1239401007572029440     1.239401e+18  1584331201000  2020-03-16   \n",
       "\n",
       "                                                    tweet hashtags  \\\n",
       "3       The new police issue masks might not prevent c...       []   \n",
       "11      Those *who are with the virus* should wear a m...       []   \n",
       "15      … I guess Nancy can put in masks for everyone ...       []   \n",
       "23      Washing your hands and disinfecting your elect...       []   \n",
       "27      Americans worried about coronavirus don't need...       []   \n",
       "...                                                   ...      ...   \n",
       "736981  corona virus finally giving me a reason to dro...       []   \n",
       "736984  2 questions: \\n\\nIs this a good time to finish...       []   \n",
       "736985  The CDC says masks don't work. The virus is to...       []   \n",
       "736986  Those who have hoarded alcohol, masks, disinfe...       []   \n",
       "736987  Doctor Oz said people are wearing surgical mas...       []   \n",
       "\n",
       "             user_id         username                               name  \\\n",
       "3       3.931759e+09     hueman_being                         Ken Parker   \n",
       "11      2.087783e+07         danimadu          manequim da guarita da PF   \n",
       "15      2.846698e+09  We4EverOvercome  Not Worthy Of Anything Remarkable   \n",
       "23      2.756134e+09   monicam_mendez                    𝑴𝒐𝒏𝒊𝒄𝒂 𝑴𝒆𝒏𝒅𝒆𝒛 🌙   \n",
       "27      3.161490e+09           gtj654                        Guy Johnson   \n",
       "...              ...              ...                                ...   \n",
       "736981  2.589109e+09         jonwadec                                jon   \n",
       "736984  4.880158e+09     70bpmhiccups                        Joe Seatter   \n",
       "736985  3.377558e+09       GunSensePA                       Gun Sense PA   \n",
       "736986  2.664186e+08         maricapz                    Marie Capanzana   \n",
       "736987  2.543078e+07        jasdagod_                         JAS DA GOD   \n",
       "\n",
       "                                                     link retweet nlikes  \\\n",
       "3       https://twitter.com/hueman_being/status/123426...   False      0   \n",
       "11      https://twitter.com/danimadu/status/1234267074...   False      0   \n",
       "15      https://twitter.com/We4EverOvercome/status/123...   False      0   \n",
       "23      https://twitter.com/monicam_mendez/status/1234...   False      6   \n",
       "27      https://twitter.com/gtj654/status/123426692152...   False      1   \n",
       "...                                                   ...     ...    ...   \n",
       "736981  https://twitter.com/jonwadec/status/1239401044...   False      8   \n",
       "736984  https://twitter.com/70bpmhiccups/status/123940...   False      1   \n",
       "736985  https://twitter.com/GunSensePA/status/12394010...   False      1   \n",
       "736986  https://twitter.com/maricapz/status/1239401009...   False      0   \n",
       "736987  https://twitter.com/jasdagod_/status/123940100...   False      0   \n",
       "\n",
       "        nreplies  nretweets  \\\n",
       "3            0.0        0.0   \n",
       "11           0.0        0.0   \n",
       "15           0.0        0.0   \n",
       "23           0.0        1.0   \n",
       "27           0.0        0.0   \n",
       "...          ...        ...   \n",
       "736981       1.0        0.0   \n",
       "736984       0.0        0.0   \n",
       "736985       1.0        0.0   \n",
       "736986       0.0        0.0   \n",
       "736987       0.0        0.0   \n",
       "\n",
       "                                                quote_url  retweet_id  \\\n",
       "3                                                     NaN         NaN   \n",
       "11                                                    NaN         NaN   \n",
       "15                                                    NaN         NaN   \n",
       "23      https://twitter.com/Pog_llins/status/123386973...         NaN   \n",
       "27                                                    NaN         NaN   \n",
       "...                                                   ...         ...   \n",
       "736981                                                NaN         NaN   \n",
       "736984                                                NaN         NaN   \n",
       "736985                                                NaN         NaN   \n",
       "736986                                                NaN         NaN   \n",
       "736987                                                NaN         NaN   \n",
       "\n",
       "                                                 reply_to  \n",
       "3       [{'user_id': '3931759154', 'username': 'hueman...  \n",
       "11      [{'user_id': '20877834', 'username': 'danimadu...  \n",
       "15      [{'user_id': '2846698472', 'username': 'We4Eve...  \n",
       "23      [{'user_id': '2756133519', 'username': 'monica...  \n",
       "27      [{'user_id': '3161490206', 'username': 'gtj654'}]  \n",
       "...                                                   ...  \n",
       "736981  [{'user_id': '2589108997', 'username': 'jonwad...  \n",
       "736984  [{'user_id': '4880157513', 'username': '70bpmh...  \n",
       "736985  [{'user_id': '3377557565', 'username': 'GunSen...  \n",
       "736986  [{'user_id': '266418579', 'username': 'maricap...  \n",
       "736987  [{'user_id': '25430780', 'username': 'jasdagod...  \n",
       "\n",
       "[188691 rows x 17 columns]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filt = 'covid|dead|death|doctor|infect|novel|nurse|outbreak|rona|sars|viral|virus|wuhan'\n",
    "mar01_clean[mar01_clean['tweet'].str.contains(filt)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"“Doctor's advice about coronavirus:\\n\\n1. It is pretty large in size (cell is about 400-500nm diameter), so any normal mask (not just the  N95 feature) should be able to filter it out.” 1/10\""
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean[jan_clean['tweet'].str.contains('corona')].loc[476743]['tweet']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' http://www.nicovideo.jp/watch/nm11194952\\xa0…\\u3000Hey Pachuco! / The Mask'"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean.loc[62841]['tweet']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2020-01-01      92\n",
       "2020-01-02     157\n",
       "2020-01-03     158\n",
       "2020-01-04     157\n",
       "2020-01-05     177\n",
       "2020-01-06     200\n",
       "2020-01-07     166\n",
       "2020-01-08     176\n",
       "2020-01-09     183\n",
       "2020-01-10     168\n",
       "2020-01-11     127\n",
       "2020-01-12     220\n",
       "2020-01-13     180\n",
       "2020-01-14     195\n",
       "2020-01-15     190\n",
       "2020-01-16     150\n",
       "2020-01-17     162\n",
       "2020-01-18     124\n",
       "2020-01-19     115\n",
       "2020-01-20     353\n",
       "2020-01-21     739\n",
       "2020-01-22    1209\n",
       "2020-01-23    2112\n",
       "2020-01-24    2441\n",
       "2020-01-25    3418\n",
       "2020-01-26    4001\n",
       "2020-01-27    4069\n",
       "2020-01-28    5240\n",
       "2020-01-29    5453\n",
       "2020-01-30    8472\n",
       "2020-01-31    7551\n",
       "Name: tweet, dtype: int64"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean[jan_clean['tweet'].str.contains(filt)].groupby('date')['tweet'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2020-02-01     4190\n",
       "2020-02-02     4018\n",
       "2020-02-03     3183\n",
       "2020-02-04     3075\n",
       "2020-02-05     2384\n",
       "2020-02-06     2778\n",
       "2020-02-07     3355\n",
       "2020-02-08     2444\n",
       "2020-02-09     2033\n",
       "2020-02-10     2546\n",
       "2020-02-11     2226\n",
       "2020-02-12     1943\n",
       "2020-02-13     2196\n",
       "2020-02-14     2087\n",
       "2020-02-15     1484\n",
       "2020-02-16     1812\n",
       "2020-02-17     7299\n",
       "2020-02-18     1645\n",
       "2020-02-19     1516\n",
       "2020-02-20     1426\n",
       "2020-02-21     1824\n",
       "2020-02-22     1627\n",
       "2020-02-23     2075\n",
       "2020-02-24     3055\n",
       "2020-02-25     4286\n",
       "2020-02-26     7537\n",
       "2020-02-27    10243\n",
       "2020-02-28    11019\n",
       "2020-02-29     9213\n",
       "Name: tweet, dtype: int64"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_clean[feb_clean['tweet'].str.contains(filt)].groupby('date')['tweet'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2020-03-01     8592\n",
       "2020-03-02    10787\n",
       "2020-03-03     9132\n",
       "2020-03-04    11386\n",
       "2020-03-05    11790\n",
       "2020-03-06     8916\n",
       "2020-03-07     8141\n",
       "2020-03-08     6661\n",
       "2020-03-09     9877\n",
       "2020-03-10     9360\n",
       "2020-03-11    18298\n",
       "2020-03-12    14898\n",
       "2020-03-13    16256\n",
       "2020-03-14    14777\n",
       "2020-03-15    13820\n",
       "2020-03-16    16000\n",
       "Name: tweet, dtype: int64"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar01_clean[mar01_clean['tweet'].str.contains(filt)].groupby('date')['tweet'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pair Down to 5k Tweets per Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((450464, 17), (536865, 17), (830073, 17), (297785, 17))"
      ]
     },
     "execution_count": 463,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_clean = pd.read_csv('Datasets/jan_tweets_clean.csv', index_col=0)\n",
    "feb_clean = pd.read_csv('Datasets/feb_tweets_clean.csv', index_col=0)\n",
    "mar_clean_10k_from_17th = pd.read_csv('Datasets/mar_tweets_clean_10k_from_17th.csv', index_col=0)\n",
    "apr_clean_10k = pd.read_csv('Datasets/apr_tweets_clean_10k.csv', index_col=0)\n",
    "\n",
    "jan_clean.shape, feb_clean.shape, mar_clean_10k_from_17th.shape, apr_clean_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((40724, 17), (78094, 17), (152131, 17), (150000, 17))"
      ]
     },
     "execution_count": 464,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filt = 'covid|dead|death|doctor|infect|novel|nurse|outbreak|rona|sars|viral|virus|wuhan'\n",
    "\n",
    "jan_5k = jan_clean[jan_clean['tweet'].str.contains(filt)].groupby('date').head(5000)\n",
    "jan_5k.reset_index(drop=True, inplace=True)\n",
    "\n",
    "feb_5k = feb_clean[feb_clean['tweet'].str.contains(filt)].groupby('date').head(5000)\n",
    "feb_5k.reset_index(drop=True, inplace=True)\n",
    "\n",
    "mar_5k = mar_clean_10k_from_17th[mar_clean_10k_from_17th['tweet'].str.contains(filt)].groupby('date').head(5000)\n",
    "mar_5k.reset_index(drop=True, inplace=True)\n",
    "\n",
    "apr_5k = apr_clean_10k[apr_clean_10k['tweet'].str.contains(filt)].groupby('date').head(5000)\n",
    "apr_5k.reset_index(drop=True, inplace=True)\n",
    "\n",
    "jan_5k.shape, feb_5k.shape, mar_5k.shape, apr_5k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2020-01-01      91\n",
       "2020-01-02     157\n",
       "2020-01-03     156\n",
       "2020-01-04     155\n",
       "2020-01-05     174\n",
       "2020-01-06     192\n",
       "2020-01-07     164\n",
       "2020-01-08     168\n",
       "2020-01-09     178\n",
       "2020-01-10     163\n",
       "2020-01-11     120\n",
       "2020-01-12     215\n",
       "2020-01-13     175\n",
       "2020-01-14     192\n",
       "2020-01-15     188\n",
       "2020-01-16     144\n",
       "2020-01-17     156\n",
       "2020-01-18     124\n",
       "2020-01-19     110\n",
       "2020-01-20     347\n",
       "2020-01-21     725\n",
       "2020-01-22    1183\n",
       "2020-01-23    1985\n",
       "2020-01-24    2331\n",
       "2020-01-25    3324\n",
       "2020-01-26    3879\n",
       "2020-01-27    3928\n",
       "2020-01-28    5000\n",
       "2020-01-29    5000\n",
       "2020-01-30    5000\n",
       "2020-01-31    5000\n",
       "Name: tweet, dtype: int64"
      ]
     },
     "execution_count": 465,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_5k[jan_5k.tweet.str.contains(filt)].groupby('date')['tweet'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2020-02-01    4057\n",
       "2020-02-02    3792\n",
       "2020-02-03    3080\n",
       "2020-02-04    2960\n",
       "2020-02-05    2309\n",
       "2020-02-06    2614\n",
       "2020-02-07    3202\n",
       "2020-02-08    2322\n",
       "2020-02-09    1914\n",
       "2020-02-10    2450\n",
       "2020-02-11    2131\n",
       "2020-02-12    1875\n",
       "2020-02-13    2135\n",
       "2020-02-14    1948\n",
       "2020-02-15    1398\n",
       "2020-02-16    1719\n",
       "2020-02-17    1540\n",
       "2020-02-18    1536\n",
       "2020-02-19    1415\n",
       "2020-02-20    1349\n",
       "2020-02-21    1760\n",
       "2020-02-22    1563\n",
       "2020-02-23    1993\n",
       "2020-02-24    2877\n",
       "2020-02-25    4155\n",
       "2020-02-26    5000\n",
       "2020-02-27    5000\n",
       "2020-02-28    5000\n",
       "2020-02-29    5000\n",
       "Name: tweet, dtype: int64"
      ]
     },
     "execution_count": 466,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feb_5k[feb_5k.tweet.str.contains(filt)].groupby('date')['tweet'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 467,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2020-03-01    5000\n",
       "2020-03-02    5000\n",
       "2020-03-03    5000\n",
       "2020-03-04    5000\n",
       "2020-03-05    5000\n",
       "2020-03-06    5000\n",
       "2020-03-07    5000\n",
       "2020-03-08    5000\n",
       "2020-03-09    5000\n",
       "2020-03-10    5000\n",
       "2020-03-11    5000\n",
       "2020-03-12    5000\n",
       "2020-03-13    5000\n",
       "2020-03-14    5000\n",
       "2020-03-15    5000\n",
       "2020-03-16    5000\n",
       "2020-03-17    5000\n",
       "2020-03-18    5000\n",
       "2020-03-19    5000\n",
       "2020-03-20    5000\n",
       "2020-03-21    5000\n",
       "2020-03-22    5000\n",
       "2020-03-23    5000\n",
       "2020-03-24    5000\n",
       "2020-03-25    5000\n",
       "2020-03-26    5000\n",
       "2020-03-27    5000\n",
       "2020-03-28    2131\n",
       "2020-03-29    5000\n",
       "2020-03-30    5000\n",
       "2020-03-31    5000\n",
       "Name: tweet, dtype: int64"
      ]
     },
     "execution_count": 467,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar_5k[mar_5k.tweet.str.contains(filt)].groupby('date')['tweet'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3441, 17)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar28_10k = pd.read_csv('10k_Datasets/10k_march28_tweets.csv', index_col=0)\n",
    "mar28_10k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Not wearing masks to protect against coronavirus is a ‘big mistake,’ top Chinese scientist says  https://www.sciencemag.org/news/2020/03/not-wearing-masks-protect-against-coronavirus-big-mistake-top-chinese-scientist-says …                                                                              39\n",
       "Not wearing masks to protect against coronavirus is a ‘big mistake,’ top Chinese scientist says | Science | AAAS  https://www.sciencemag.org/news/2020/03/not-wearing-masks-protect-against-coronavirus-big-mistake-top-chinese-scientist-says …                                                             16\n",
       "Future's foundation is making masks for healthcare workers and patients affected by COVID-19   https://twitter.com/i/events/1243942042054475776 …                                                                                                                                                             9\n",
       "Experts say face masks can help slow COVID-19, despite previous claims  https://nypost.com/2020/03/28/experts-say-face-masks-can-help-slow-covid-19-despite-previous-claims/?utm_source=twitter_sitebuttons&utm_medium=site%20buttons&utm_campaign=site%20buttons … via @nypost                               8\n",
       "1) Follow @Mimosamania on here & IG. \\n\\n2 A) For the Nike gift card: do the savage challenge dance on live with mimosamania \\n\\n2 B) For the corona mask: do the Southside on live with mimosamania. \\n\\n2 C) For the custom dyed Nike socks: do the dougie on live with mimosamania.                        8\n",
       "                                                                                                                                                                                                                                                                                                             ..\n",
       "@NYGovCuomo STOP THE DROP! Mr. governor. Mandatory mask will significantly reduce the spread of Coronavirus. Sick people without symptoms won’t be able to spread droplets and healthy people won’t inhale droplets. This is the only way to combat the spread. We need to stop the drop                      1\n",
       "Keeping Covid-19 deaths below 20,000 would be a good result, says NHS medical director Stephen Powis who says 170million masks, 25million gloves and 30million aprons have been delivered to = 20000 might die? Not good enough the people need more info.                                                    1\n",
       "@realDonaldTrump why is the American Red Cross asking Americans to donate blood during this COVID-19 pandemic? We have been forced to self-quarantine, leave our jobs, laid off, no school, wear masks, & fear stricken ppl demanding authoritarian totalitarian martial law.  pic.twitter.com/pUZDj25wXl     1\n",
       "Saying \"N95 only\" also not quite accurate. Standard PPE in health care for covid is surgical mask(plus all other type of PPE). This is because droplet spread, not airborne.  Unless certain procedures and circumstances where might generate airborne.                                                      1\n",
       "TAKE ACTION #COVID19: Do not buy PPE that you do not need. Medical professionals need those gloves & masks. Who needs PPE:\\n\\n- If you are healthy, you only need to wear a mask if you are taking care of a person with suspected COVID19.\\n- Wear a mask if you are coughing/sneezing.                      1\n",
       "Name: tweet, Length: 3270, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mar28_10k['tweet'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2020-04-01    5000\n",
       "2020-04-02    5000\n",
       "2020-04-03    5000\n",
       "2020-04-04    5000\n",
       "2020-04-05    5000\n",
       "2020-04-06    5000\n",
       "2020-04-07    5000\n",
       "2020-04-08    5000\n",
       "2020-04-09    5000\n",
       "2020-04-10    5000\n",
       "2020-04-11    5000\n",
       "2020-04-12    5000\n",
       "2020-04-13    5000\n",
       "2020-04-14    5000\n",
       "2020-04-15    5000\n",
       "2020-04-16    5000\n",
       "2020-04-17    5000\n",
       "2020-04-18    5000\n",
       "2020-04-19    5000\n",
       "2020-04-20    5000\n",
       "2020-04-21    5000\n",
       "2020-04-22    5000\n",
       "2020-04-23    5000\n",
       "2020-04-24    5000\n",
       "2020-04-25    5000\n",
       "2020-04-26    5000\n",
       "2020-04-27    5000\n",
       "2020-04-28    5000\n",
       "2020-04-29    5000\n",
       "2020-04-30    5000\n",
       "Name: tweet, dtype: int64"
      ]
     },
     "execution_count": 468,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apr_5k[apr_5k.tweet.str.contains(filt)].groupby('date')['tweet'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [],
   "source": [
    "jan_5k.to_csv('5k_Datasets/5k_jan_tweets.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {},
   "outputs": [],
   "source": [
    "feb_5k.to_csv('5k_Datasets/5k_feb_tweets.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {},
   "outputs": [],
   "source": [
    "mar_5k.to_csv('5k_Datasets/5k_mar_tweets.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 469,
   "metadata": {},
   "outputs": [],
   "source": [
    "apr_5k.to_csv('5k_Datasets/5k_apr_tweets.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine 5k/Day Tweet Dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((40724, 17), (78094, 17), (152131, 17), (150000, 17), (155000, 15))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_5k = pd.read_csv('5k_Datasets/5k_jan_tweets.csv', index_col=0)\n",
    "\n",
    "feb_5k = pd.read_csv('5k_Datasets/5k_feb_tweets.csv', index_col=0)\n",
    "\n",
    "mar_5k = pd.read_csv('5k_Datasets/5k_mar_tweets.csv', index_col=0)\n",
    "\n",
    "apr_5k = pd.read_csv('5k_Datasets/5k_apr_tweets.csv', index_col=0)\n",
    "\n",
    "may_5k = pd.read_csv('5k_Datasets/5k_may_tweets.csv', index_col=0)\n",
    "\n",
    "jan_5k.shape, feb_5k.shape, mar_5k.shape, apr_5k.shape, may_5k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(575949, 17)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([jan_5k, feb_5k, mar_5k, apr_5k, may_5k])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.212523e+18</td>\n",
       "      <td>1.212523e+18</td>\n",
       "      <td>1.577923e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>https://mltshp.com/p/1HLSB  \"This is the deat...</td>\n",
       "      <td>[]</td>\n",
       "      <td>8.431951e+17</td>\n",
       "      <td>Best_of_MLTSHP</td>\n",
       "      <td>MLTSHP</td>\n",
       "      <td>https://twitter.com/Best_of_MLTSHP/status/1212...</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '843195118924546048', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.212518e+18</td>\n",
       "      <td>1.212518e+18</td>\n",
       "      <td>1.577922e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Ready to heard this? Homemade hair treatment a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.044739e+18</td>\n",
       "      <td>lumora_lu</td>\n",
       "      <td>MiLu</td>\n",
       "      <td>https://twitter.com/lumora_lu/status/121251804...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1044738746913034241', 'username'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.212518e+18</td>\n",
       "      <td>1.212518e+18</td>\n",
       "      <td>1.577922e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>We all know what a joke #CCPChina is when it c...</td>\n",
       "      <td>['#ccpchina', '#sars', '#hk', '#hkers']</td>\n",
       "      <td>9.630325e+17</td>\n",
       "      <td>odiecher</td>\n",
       "      <td>odiecher</td>\n",
       "      <td>https://twitter.com/odiecher/status/1212517557...</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://twitter.com/WBYeats1865/status/1212430...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '963032516759207936', 'username':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.212516e+18</td>\n",
       "      <td>1.200523e+18</td>\n",
       "      <td>1.577921e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Thought death masks gave you the horn freak</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.819219e+09</td>\n",
       "      <td>Hevysmoker</td>\n",
       "      <td>Ashley Collins</td>\n",
       "      <td>https://twitter.com/Hevysmoker/status/12125155...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '2819219369', 'username': 'Hevysm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.212510e+18</td>\n",
       "      <td>1.212391e+18</td>\n",
       "      <td>1.577920e+12</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Interesting cult of death mask he’s wearing, 🤔...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.361956e+08</td>\n",
       "      <td>FaithR8s</td>\n",
       "      <td>❌Patriot Fan🇺🇸🗽☕️🎶❌</td>\n",
       "      <td>https://twitter.com/FaithR8s/status/1212510409...</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '236195607', 'username': 'FaithR8...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  conversation_id    created_at        date  \\\n",
       "0  1.212523e+18     1.212523e+18  1.577923e+12  2020-01-01   \n",
       "1  1.212518e+18     1.212518e+18  1.577922e+12  2020-01-01   \n",
       "2  1.212518e+18     1.212518e+18  1.577922e+12  2020-01-01   \n",
       "3  1.212516e+18     1.200523e+18  1.577921e+12  2020-01-01   \n",
       "4  1.212510e+18     1.212391e+18  1.577920e+12  2020-01-01   \n",
       "\n",
       "                                               tweet  \\\n",
       "0   https://mltshp.com/p/1HLSB  \"This is the deat...   \n",
       "1  Ready to heard this? Homemade hair treatment a...   \n",
       "2  We all know what a joke #CCPChina is when it c...   \n",
       "3        Thought death masks gave you the horn freak   \n",
       "4  Interesting cult of death mask he’s wearing, 🤔...   \n",
       "\n",
       "                                  hashtags       user_id        username  \\\n",
       "0                                       []  8.431951e+17  Best_of_MLTSHP   \n",
       "1                                       []  1.044739e+18       lumora_lu   \n",
       "2  ['#ccpchina', '#sars', '#hk', '#hkers']  9.630325e+17        odiecher   \n",
       "3                                       []  2.819219e+09      Hevysmoker   \n",
       "4                                       []  2.361956e+08        FaithR8s   \n",
       "\n",
       "                  name                                               link  \\\n",
       "0               MLTSHP  https://twitter.com/Best_of_MLTSHP/status/1212...   \n",
       "1                 MiLu  https://twitter.com/lumora_lu/status/121251804...   \n",
       "2             odiecher  https://twitter.com/odiecher/status/1212517557...   \n",
       "3       Ashley Collins  https://twitter.com/Hevysmoker/status/12125155...   \n",
       "4  ❌Patriot Fan🇺🇸🗽☕️🎶❌  https://twitter.com/FaithR8s/status/1212510409...   \n",
       "\n",
       "   retweet  nlikes  nreplies  nretweets  \\\n",
       "0    False     1.0       0.0        1.0   \n",
       "1    False     0.0       0.0        0.0   \n",
       "2    False     1.0       1.0        1.0   \n",
       "3    False     0.0       0.0        0.0   \n",
       "4    False     1.0       0.0        1.0   \n",
       "\n",
       "                                           quote_url  retweet_id  \\\n",
       "0                                                NaN         NaN   \n",
       "1                                                NaN         NaN   \n",
       "2  https://twitter.com/WBYeats1865/status/1212430...         NaN   \n",
       "3                                                NaN         NaN   \n",
       "4                                                NaN         NaN   \n",
       "\n",
       "                                            reply_to  \n",
       "0  [{'user_id': '843195118924546048', 'username':...  \n",
       "1  [{'user_id': '1044738746913034241', 'username'...  \n",
       "2  [{'user_id': '963032516759207936', 'username':...  \n",
       "3  [{'user_id': '2819219369', 'username': 'Hevysm...  \n",
       "4  [{'user_id': '236195607', 'username': 'FaithR8...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.reset_index(drop=True, inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>575944</th>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.590909e+12</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>Cuomo urges protesters to 'demonstrate with a ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.424594e+08</td>\n",
       "      <td>chrisusanz</td>\n",
       "      <td>Christopher</td>\n",
       "      <td>https://twitter.com/chrisusanz/status/12669917...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '342459363', 'username': 'chrisus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575945</th>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.590909e+12</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>Advantages of wearing face masks: \\n1. Haven’t...</td>\n",
       "      <td>['#covid19', '#coronapandemic', '#sundayvibes']</td>\n",
       "      <td>1.361908e+09</td>\n",
       "      <td>HusaynZ</td>\n",
       "      <td>Rogue Aviator 🇵🇰</td>\n",
       "      <td>https://twitter.com/HusaynZ/status/12669916005...</td>\n",
       "      <td>False</td>\n",
       "      <td>62.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '1361908454', 'username': 'Husayn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575946</th>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.590909e+12</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>The model of South Korea will have to be follo...</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.223174e+09</td>\n",
       "      <td>moussetafaa</td>\n",
       "      <td>Mustafa</td>\n",
       "      <td>https://twitter.com/moussetafaa/status/1266991...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '3223174388', 'username': 'mousse...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575947</th>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.266992e+18</td>\n",
       "      <td>1.590909e+12</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>Aeroflot: passengers must change face masks ev...</td>\n",
       "      <td>['#aviation', '#airline', '#aircraft', '#covid...</td>\n",
       "      <td>1.545939e+07</td>\n",
       "      <td>SpeedBird_NCL</td>\n",
       "      <td>SpeedBird</td>\n",
       "      <td>https://twitter.com/SpeedBird_NCL/status/12669...</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '15459391', 'username': 'SpeedBir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575948</th>\n",
       "      <td>1.266991e+18</td>\n",
       "      <td>1.266991e+18</td>\n",
       "      <td>1.590909e+12</td>\n",
       "      <td>2020-05-31</td>\n",
       "      <td>Corona said mask on 2020 said fuck it mask off...</td>\n",
       "      <td>[]</td>\n",
       "      <td>7.181234e+17</td>\n",
       "      <td>sharunbelll</td>\n",
       "      <td>moona love ❣️</td>\n",
       "      <td>https://twitter.com/sharunbelll/status/1266991...</td>\n",
       "      <td>False</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'user_id': '718123376804376576', 'username':...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id  conversation_id    created_at        date  \\\n",
       "575944  1.266992e+18     1.266992e+18  1.590909e+12  2020-05-31   \n",
       "575945  1.266992e+18     1.266992e+18  1.590909e+12  2020-05-31   \n",
       "575946  1.266992e+18     1.266992e+18  1.590909e+12  2020-05-31   \n",
       "575947  1.266992e+18     1.266992e+18  1.590909e+12  2020-05-31   \n",
       "575948  1.266991e+18     1.266991e+18  1.590909e+12  2020-05-31   \n",
       "\n",
       "                                                    tweet  \\\n",
       "575944  Cuomo urges protesters to 'demonstrate with a ...   \n",
       "575945  Advantages of wearing face masks: \\n1. Haven’t...   \n",
       "575946  The model of South Korea will have to be follo...   \n",
       "575947  Aeroflot: passengers must change face masks ev...   \n",
       "575948  Corona said mask on 2020 said fuck it mask off...   \n",
       "\n",
       "                                                 hashtags       user_id  \\\n",
       "575944                                                 []  3.424594e+08   \n",
       "575945    ['#covid19', '#coronapandemic', '#sundayvibes']  1.361908e+09   \n",
       "575946                                                 []  3.223174e+09   \n",
       "575947  ['#aviation', '#airline', '#aircraft', '#covid...  1.545939e+07   \n",
       "575948                                                 []  7.181234e+17   \n",
       "\n",
       "             username              name  \\\n",
       "575944     chrisusanz       Christopher   \n",
       "575945        HusaynZ  Rogue Aviator 🇵🇰   \n",
       "575946    moussetafaa           Mustafa   \n",
       "575947  SpeedBird_NCL         SpeedBird   \n",
       "575948    sharunbelll     moona love ❣️   \n",
       "\n",
       "                                                     link  retweet  nlikes  \\\n",
       "575944  https://twitter.com/chrisusanz/status/12669917...    False     0.0   \n",
       "575945  https://twitter.com/HusaynZ/status/12669916005...    False    62.0   \n",
       "575946  https://twitter.com/moussetafaa/status/1266991...    False     0.0   \n",
       "575947  https://twitter.com/SpeedBird_NCL/status/12669...    False     1.0   \n",
       "575948  https://twitter.com/sharunbelll/status/1266991...    False     1.0   \n",
       "\n",
       "        nreplies  nretweets quote_url  retweet_id  \\\n",
       "575944       0.0        0.0       NaN         NaN   \n",
       "575945      10.0       11.0       NaN         NaN   \n",
       "575946       1.0        0.0       NaN         NaN   \n",
       "575947       0.0        0.0       NaN         NaN   \n",
       "575948       0.0        1.0       NaN         NaN   \n",
       "\n",
       "                                                 reply_to  \n",
       "575944  [{'user_id': '342459363', 'username': 'chrisus...  \n",
       "575945  [{'user_id': '1361908454', 'username': 'Husayn...  \n",
       "575946  [{'user_id': '3223174388', 'username': 'mousse...  \n",
       "575947  [{'user_id': '15459391', 'username': 'SpeedBir...  \n",
       "575948  [{'user_id': '718123376804376576', 'username':...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        2020-01-01\n",
       "1        2020-01-01\n",
       "2        2020-01-01\n",
       "3        2020-01-01\n",
       "4        2020-01-01\n",
       "            ...    \n",
       "575944   2020-05-31\n",
       "575945   2020-05-31\n",
       "575946   2020-05-31\n",
       "575947   2020-05-31\n",
       "575948   2020-05-31\n",
       "Name: date, Length: 575949, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%d')\n",
    "df['date']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}