data_pull.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "''' Modified from lukefeilberg's code here: https://github.com/lukefeilberg/onion'''\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "import re\n",
    "import time\n",
    "import requests\n",
    "import json\n",
    "import csv\n",
    "import datetime\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['01/01/2010',\n '01/03/2010',\n '01/05/2010',\n '01/07/2010',\n '01/09/2010',\n '01/12/2010',\n '01/01/2011',\n '01/03/2011',\n '01/05/2011',\n '01/07/2011',\n '01/09/2011',\n '01/12/2011',\n '01/01/2012',\n '01/03/2012',\n '01/05/2012',\n '01/07/2012',\n '01/09/2012',\n '01/12/2012',\n '01/01/2013',\n '01/03/2013',\n '01/05/2013',\n '01/07/2013',\n '01/09/2013',\n '01/12/2013',\n '01/01/2014',\n '01/03/2014',\n '01/05/2014',\n '01/07/2014',\n '01/09/2014',\n '01/12/2014',\n '01/01/2015',\n '01/03/2015',\n '01/05/2015',\n '01/07/2015',\n '01/09/2015',\n '01/12/2015',\n '01/01/2016',\n '01/03/2016',\n '01/05/2016',\n '01/07/2016',\n '01/09/2016',\n '01/12/2016',\n '01/01/2017',\n '01/03/2017',\n '01/05/2017',\n '01/07/2017',\n '01/09/2017',\n '01/12/2017',\n '01/01/2018',\n '01/03/2018',\n '01/05/2018',\n '01/07/2018',\n '01/09/2018',\n '01/12/2018',\n '01/01/2019']"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 2
    }
   ],
   "source": [
    "dates_list = []\n",
    "\n",
    "# Making list of dates; Each January 1st and June 1st from 2015 until January 1st 2020\n",
    "for i in range(10,21):\n",
    "    dates_list.append('01/01/20'+str(i))\n",
    "    # dates_list.append('01/02/20'+str(i))\n",
    "    dates_list.append('01/03/20'+str(i))\n",
    "    # dates_list.append('01/04/20'+str(i))\n",
    "    dates_list.append('01/05/20'+str(i))\n",
    "    # dates_list.append('01/06/20'+str(i))\n",
    "    dates_list.append('01/07/20'+str(i))\n",
    "    # dates_list.append('01/08/20'+str(i))\n",
    "    dates_list.append('01/09/20'+str(i))\n",
    "    # dates_list.append('01/10/20'+str(i))\n",
    "    # dates_list.append('01/11/20'+str(i))\n",
    "    dates_list.append('01/12/20'+str(i))\n",
    "\n",
    "dates_list = dates_list[:-11]    \n",
    "dates_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def getPushshiftData(after, before, sub):\n",
    "    url = ('https://api.pushshift.io/reddit/search/submission/?size=1000&after='+\n",
    "           str(after)+'&before='+str(before)+'&subreddit='+str(sub)+'&sort_type=score'+'&sort=desc')\n",
    "    r = requests.get(url)\n",
    "    data = json.loads(r.text)\n",
    "    return data['data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "[1262325600,\n 1267423200,\n 1272690000,\n 1277960400,\n 1283317200,\n 1291183200,\n 1293861600,\n 1298959200,\n 1304226000,\n 1309496400,\n 1314853200,\n 1322719200,\n 1325397600,\n 1330581600,\n 1335848400,\n 1341118800,\n 1346475600,\n 1354341600,\n 1357020000,\n 1362117600,\n 1367384400,\n 1372654800,\n 1378011600,\n 1385877600,\n 1388556000,\n 1393653600,\n 1398920400,\n 1404190800,\n 1409547600,\n 1417413600,\n 1420092000,\n 1425189600,\n 1430456400,\n 1435726800,\n 1441083600,\n 1448949600,\n 1451628000,\n 1456812000,\n 1462078800,\n 1467349200,\n 1472706000,\n 1480572000,\n 1483250400,\n 1488348000,\n 1493614800,\n 1498885200,\n 1504242000,\n 1512108000,\n 1514786400,\n 1519884000,\n 1525150800,\n 1530421200,\n 1535778000,\n 1543644000,\n 1546322400]"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 4
    }
   ],
   "source": [
    "def getTimeStamp(date_input):\n",
    "    return time.mktime(datetime.datetime.strptime(date_input, \"%d/%m/%Y\").timetuple())\n",
    "\n",
    "dates = [int(getTimeStamp(date)) for date in dates_list]\n",
    "dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def getTitles(subreddit):\n",
    "    titles = []\n",
    "\n",
    "    for i in tqdm(range(len(dates)-1)):\n",
    "        # Setting up dates\n",
    "        after  = dates[i]\n",
    "        before = dates[i+1]\n",
    "\n",
    "        # Getting subreddit data between the dates after and before from r/NotTheOnion\n",
    "        raw_json = getPushshiftData(after,before,subreddit)\n",
    "\n",
    "        # Extracting just the title\n",
    "        \n",
    "        titles_new = [post['title'] for post in raw_json]\n",
    "\n",
    "        # Appending new data on\n",
    "        titles = titles + titles_new\n",
    "        \n",
    "        time.sleep(1)\n",
    "\n",
    "    # A few posts were extracted twice, set gets rid of duplicates\n",
    "    titles = list(set(titles))\n",
    "    return titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "100%|██████████| 54/54 [02:02<00:00,  2.27s/it]\n"
     ],
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": [
      "38841\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "not_onion = getTitles('nottheonion')\n",
    "print(len(not_onion))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 120/120 [03:53<00:00,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15293\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "onion = getTitles('theonion')\n",
    "print(len(onion))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Comcast Whines About How Net Neutrality Is Lik...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chattanooga Wants You to Know That Iron Man 3 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Men who buried dog alive with nail in its head...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Massive Swirl Of Ocean Debris To Be Designated...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Half-naked man makes Subway sandwiches in ‘con...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995</th>\n",
       "      <td>Confused Japanese tourists trigger highway pur...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14996</th>\n",
       "      <td>‘Dead’ Teen ‘Wakes Up’ Screaming, Banging Insi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14997</th>\n",
       "      <td>Congresswoman says pornography is a root cause...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14998</th>\n",
       "      <td>Russian Woman Eats a Pig to Avoid Paying Child...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14999</th>\n",
       "      <td>The Verge: Industrial band Skinny Puppy bills ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label\n",
       "0      Comcast Whines About How Net Neutrality Is Lik...      0\n",
       "1      Chattanooga Wants You to Know That Iron Man 3 ...      0\n",
       "2      Men who buried dog alive with nail in its head...      0\n",
       "3      Massive Swirl Of Ocean Debris To Be Designated...      0\n",
       "4      Half-naked man makes Subway sandwiches in ‘con...      0\n",
       "...                                                  ...    ...\n",
       "14995  Confused Japanese tourists trigger highway pur...      0\n",
       "14996  ‘Dead’ Teen ‘Wakes Up’ Screaming, Banging Insi...      0\n",
       "14997  Congresswoman says pornography is a root cause...      0\n",
       "14998  Russian Woman Eats a Pig to Avoid Paying Child...      0\n",
       "14999  The Verge: Industrial band Skinny Puppy bills ...      0\n",
       "\n",
       "[15000 rows x 2 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1= pd.DataFrame({'text':onion})\n",
    "df1['label'] = 1\n",
    "df1 = df1.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "df2 = pd.DataFrame({'text':not_onion})\n",
    "df2['label'] = 0\n",
    "df2 = df2.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "df1_clipped = df1[:15000]\n",
    "df2_clipped = df2[:15000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combining both datasets\n",
    "df = pd.concat([df1_clipped, df2_clipped])\n",
    "\n",
    "# Shuffling the dataset\n",
    "df = df.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "# Saving the 'uncleaned' dataframe to a csv file\n",
    "df.to_csv('OnionOrNot_large.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}