StockNews2018Pkg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <font color=blue>#StockNews 2018</color>\n",
    "#### <font color=green>[by Ethan Shaotran]</color>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Load():\n",
    "    from keras.models import Sequential\n",
    "    from keras.layers import Dense\n",
    "    from keras.layers import Dropout\n",
    "    from keras.layers.normalization import BatchNormalization\n",
    "    from keras.wrappers.scikit_learn import KerasRegressor\n",
    "    from keras.models import model_from_json #Reload a model\n",
    "    #from sklearn.model_selection import cross_val_score\n",
    "    #from sklearn.model_selection import KFold\n",
    "    #from sklearn.preprocessing import StandardScaler\n",
    "    #from sklearn.pipeline import Pipeline\n",
    "    from sklearn import datasets, linear_model, preprocessing\n",
    "    from sklearn.metrics import mean_squared_error\n",
    "    #from sklearn.model_selection import train_test_split\n",
    "    from matplotlib import pyplot as plt\n",
    "    import pylab\n",
    "    import numpy as np\n",
    "    import pandas as pd\n",
    "    from pandas import read_csv\n",
    "    import requests\n",
    "    from contextlib import closing\n",
    "    import math\n",
    "    import csv\n",
    "    import os\n",
    "    from datetime import datetime, timedelta\n",
    "    from random import randint\n",
    "\n",
    "    #Alpha Vantage - Data Point Requests:\n",
    "    apikey = 'VPRQOQ85W9KS1DXH'\n",
    "\n",
    "\n",
    "    #News Articles URL Searches: - news-corpus-builder\n",
    "    from news_corpus_builder import NewsCorpusGenerator\n",
    "    corpus_dir = '.' \n",
    "    NewsGen = NewsCorpusGenerator(corpus_dir,'flat')\n",
    "\n",
    "    #News Article Text Scraping:\n",
    "    from newspaper import Article #USED LATER\n",
    "\n",
    "    #Numpy Random\n",
    "    seed = 7\n",
    "    np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [F] Collect Historical Data Points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def CollectData(Ticker):\n",
    "    import pandas as pd\n",
    "    apikey = 'VPRQOQ85W9KS1DXH'\n",
    "    url = \"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol=\" + Ticker + \"&outputsize=full&apikey=\" + apikey + \"&datatype=csv\"\n",
    "    \n",
    "    StockHistory = pd.read_csv(url)\n",
    "    StockHistory.drop(StockHistory.columns[1], axis=1, inplace=True) #1\n",
    "    StockHistory.drop(StockHistory.columns[1], axis=1, inplace=True) #2\n",
    "    StockHistory.drop(StockHistory.columns[1], axis=1, inplace=True) #3\n",
    "    StockHistory.drop(StockHistory.columns[1], axis=1, inplace=True) #4\n",
    "    StockHistory.drop(StockHistory.columns[2], axis=1, inplace=True) #6\n",
    "    StockHistory.drop(StockHistory.columns[2], axis=1, inplace=True) #7\n",
    "    StockHistory.drop(StockHistory.columns[2], axis=1, inplace=True) #8\n",
    "#    StockHistory.drop(StockHistory.columns[[1,2,3,4,6,7,8]], axis=1, inplace=True)\n",
    "    print(\"Collected data from \" + Ticker + \" for \" + str(StockHistory.shape[0]) + \" days!\")\n",
    "    StockHistory.to_csv('StockHistoryData/'+Ticker+'.csv')\n",
    "    return StockHistory\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [F] Collect News URLS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def GetNewsURLs(Ticker):\n",
    "    from news_corpus_builder import NewsCorpusGenerator\n",
    "    corpus_dir = '.' \n",
    "    NewsGen = NewsCorpusGenerator(corpus_dir,'flat')\n",
    "    links = NewsGen.google_news_search(Ticker+' stocks',100)\n",
    "    URLList=[]\n",
    "    for i in range(0,len(links)-1):\n",
    "        URLLink = links[i][1]\n",
    "        if not (SourceInt(URLLink) == 0):\n",
    "            URLList.append(URLLink)\n",
    "    print(\"Found \" + str(len(URLList)) + \" Articles for Stock \" + Ticker)\n",
    "    return URLList\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [F] Extract Text from URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def URLtoText (url):\n",
    "    from newspaper import Article\n",
    "    article = Article(url)\n",
    "    article.download()\n",
    "    article.parse()\n",
    "    articletext = article.text\n",
    "    articledate = article.publish_date\n",
    "\n",
    "    return (articletext,articledate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def URLtoTextSummary (url):\n",
    "    from newspaper import Article\n",
    "    article = Article(url)\n",
    "    article.download()\n",
    "    article.parse()\n",
    "    article.nlp()\n",
    "    articletext = article.summary\n",
    "    articledate = article.publish_date\n",
    "\n",
    "    return (articletext,articledate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [F] Get Source of URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def SourceInt (url):\n",
    "    url = url.lower()\n",
    "    if ('cnn' in url):\n",
    "        sourceint = 1\n",
    "    elif ('fool.com' in url):\n",
    "        sourceint = 2\n",
    "    elif ('finance.yahoo.com' in url):\n",
    "        sourceint = 3\n",
    "    elif ('investopedia.com' in url):\n",
    "        sourceint = 4\n",
    "    elif ('businessinsider.com' in url):\n",
    "        sourceint = 5\n",
    "    elif ('marketwatch.com' in url):\n",
    "        sourceint = 6\n",
    "    elif ('forbes.com' in url):\n",
    "        sourceint = 7\n",
    "    elif ('thestreet.com' in url):\n",
    "        sourceint = 8\n",
    "    elif ('fortune.com' in url):\n",
    "        sourceint = 9\n",
    "    elif ('cnbc.com' in url):\n",
    "        sourceint = 10\n",
    "    elif ('nasdaq.com' in url):\n",
    "        sourceint = 11\n",
    "    elif ('investorplace.com' in url):\n",
    "        sourceint = 12\n",
    "    elif ('seekingalpha.com' in url):\n",
    "        sourceint = 13\n",
    "    elif ('nytimes' in url):\n",
    "        sourceint = 14\n",
    "    elif ('cnbc' in url):\n",
    "        sourceint = 15\n",
    "    elif ('apple' in url):\n",
    "        sourceint = 16\n",
    "    elif ('techcrunch' in url):\n",
    "        sourceint = 17\n",
    "    elif ('huff' in url):\n",
    "        sourceint = 18\n",
    "    elif ('fox' in url):\n",
    "        sourceint = 19\n",
    "    elif ('usatoday' in url):\n",
    "        sourceint = 20\n",
    "    elif ('reuters' in url):\n",
    "        sourceint = 21\n",
    "    elif ('npr' in url):\n",
    "        sourceint = 22\n",
    "    elif ('nbc' in url):\n",
    "        sourceint = 23\n",
    "    elif ('fox' in url):\n",
    "        sourceint = 24\n",
    "    elif ('wsj' in url):\n",
    "        sourceint = 25\n",
    "    elif ('marketwatch' in url):\n",
    "        sourceint = 26\n",
    "    elif ('bloomberg' in url):\n",
    "        sourceint = 27\n",
    "    elif ('guardian' in url):\n",
    "        sourceint = 28\n",
    "    elif ('usnews' in url):\n",
    "        sourceint = 29\n",
    "    elif ('livetradingnews' in url):\n",
    "        sourceint = 30\n",
    "    elif ('streetinsider' in url):\n",
    "        sourceint = 31\n",
    "    elif ('stocknewsjournal' in url):\n",
    "        sourceint = 32\n",
    "    else: sourceint = 0\n",
    "    return sourceint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [F] Calculate Slope of Stock Data Points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getSlope (articledate, look_ahead, backorahead, ticker):\n",
    "    import pandas as pd\n",
    "    import math\n",
    "    StockHistory = pd.read_csv('StockHistoryData/'+ticker+'.csv')\n",
    "    \n",
    "    articledate = articledate.strftime('%Y-%m-%d')\n",
    "    t = SearchCSVForDate(articledate, ticker)\n",
    "    \n",
    "    #Calculate X avg, Create list of X Values\n",
    "    Xavg = 0\n",
    "    Xlist = []\n",
    "    if backorahead == 1:\n",
    "        for x in range(t+1,t-look_ahead,-1):\n",
    "            Xavg += x\n",
    "            Xlist.append(x)\n",
    "    else:\n",
    "        for x in range(t+look_ahead,t,-1):\n",
    "            Xavg += x\n",
    "            Xlist.append(x)\n",
    "    Xavg /= len(Xlist)\n",
    "    Xlist.reverse()\n",
    "        \n",
    "    #Calculate Y avg, Create list of Y Values\n",
    "    Yavg = 0\n",
    "    Ylist = []\n",
    "    if backorahead == 1:\n",
    "        for x in range(t+1,t-look_ahead,-1):\n",
    "            y = StockHistory.get_value(x,\"adjusted_close\")\n",
    "            Yavg += y\n",
    "            Ylist.append(y)\n",
    "    else:\n",
    "        for x in range(t+look_ahead,t,-1):\n",
    "            y = StockHistory.get_value(x,\"adjusted_close\")\n",
    "            Yavg += y\n",
    "            Ylist.append(y)\n",
    "    Yavg /= len(Ylist)\n",
    "    \n",
    "    TopCalculation = 0;\n",
    "    for i in (0,len(Xlist)-1): #Top Part\n",
    "        TopCalculation += (Xlist[i] - Xavg)*(Ylist[i] - Yavg)\n",
    "        \n",
    "    BottomCalculation = 0;\n",
    "    for i in (0,len(Xlist)-1): #Bottom Part\n",
    "        BottomCalculation += math.pow((Xlist[i] - Xavg), 2)\n",
    "        \n",
    "    Slope = TopCalculation/BottomCalculation\n",
    "        \n",
    "    return Slope;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [F] Get CSV Row # For Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def SearchCSVForDate (articledate, ticker):\n",
    "    import pandas as pd\n",
    "    StockHistory = pd.read_csv('StockHistoryData/'+ticker+'.csv')\n",
    "    return StockHistory[StockHistory['timestamp'] == articledate].index.tolist()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [F] Calculate Textual Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getSentiment (ListedText):\n",
    "    posList = []\n",
    "    negList = []\n",
    "    \n",
    "    #Calculate Positivity:\n",
    "    Positivity = 0\n",
    "    PositiveWords = open('MasterDictionaryFiles/PositiveWords.txt', 'r').read()\n",
    "    for i in range(0,len(ListedText)):\n",
    "        if ((\"_\"+ListedText[i].upper() + \"_\") in PositiveWords and not (ListedText[i] in posList)):\n",
    "            #print(\"+ \" + ListedText[i])\n",
    "            posList.append(ListedText[i])\n",
    "            Positivity += 1\n",
    "         \n",
    "    #Calculate Negativity:\n",
    "    Negativity = 0\n",
    "    NegativeWords = open('MasterDictionaryFiles/NegativeWords.txt', 'r').read()\n",
    "    for i in range(0,len(ListedText)):\n",
    "        if ((\"_\"+ListedText[i].upper() + \"_\") in NegativeWords and not (ListedText[i] in negList)):\n",
    "            #print(\"- \" + ListedText[i])\n",
    "            negList.append(ListedText[i])\n",
    "            Negativity += 1\n",
    "    \n",
    "    return Positivity, Negativity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define URLs for Training Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "URLListAAPL = [\n",
    "'http://money.cnn.com/2007/01/09/technology/apple_jobs/',\n",
    "'https://seekingalpha.com/article/50765-apples-iphone-builds-huge-lead-in-customer-satisfaction',\n",
    "'http://www.cnn.com/2008/TECH/11/18/iphone.game.developer/',\n",
    "'https://www.forbes.com/2009/01/26/steve-jobs-health-tech-enter-cz_rb_0126stevejobs.html',\n",
    "'http://www.cnn.com/2009/TECH/06/29/steve.jobs.health.work/index.html',\n",
    "'http://money.cnn.com/2010/01/27/technology/ipad_apple_stock/',\n",
    "'http://www.cnn.com/2010/TECH/ptech/04/03/apple.ipad.sale/index.html',\n",
    "'http://www.businessinsider.com/apples-1-million-ipads-in-context-2010-5',\n",
    "'http://www.marketwatch.com/story/apple-gets-boost-from-ipad-shortage-2010-04-14',\n",
    "'https://www.forbes.com/sites/heatherstruck/2011/01/17/steve-jobs-sick-leave-could-slow-down-apple-stock/#192f91ce3c02',\n",
    "'https://www.cnbc.com/id/41118102',\n",
    "'http://money.cnn.com/2011/01/18/technology/apple_jobs/index.htm',\n",
    "'http://www.businessinsider.com/steve-jobs-leave-effect-product-2011-1',\n",
    "'http://money.cnn.com/2011/07/19/technology/apple_q3_earnings/index.htm',\n",
    "'http://www.marketwatch.com/story/apple-expected-to-set-up-strong-second-half-2011-07-15',\n",
    "'http://www.marketwatch.com/story/apple-unveils-new-operating-system-for-mac-2011-06-06',\n",
    "'https://www.thestreet.com/story/11145528/1/icloud-impact-on-apple-stock.html',\n",
    "'https://investorplace.com/2011/03/tablet-wars-apple-ipad-vs-the-competition/#.WbWoYsbMz-Y',\n",
    "'https://seekingalpha.com/article/265146-the-rise-and-rise-of-apple-time-for-a-split',\n",
    "'https://seekingalpha.com/article/262746-apple-investors-time-for-caution-heading-into-q2-earnings',\n",
    "'http://www.cnn.com/2011/TECH/innovation/07/29/apple.cash.government/index.html',\n",
    "'https://www.forbes.com/sites/scottdecarlo/2011/08/11/the-worlds-25-most-valuable-companies-apple-is-now-on-top/#12ff0d993814',\n",
    "'http://www.marketwatch.com/story/apple-crowned-no-1-with-biggest-market-cap-2011-08-10',\n",
    "'https://www.forbes.com/sites/steveschaefer/2011/08/24/steve-jobs-steps-down-apple-shares-drop-5-after-hours/#2220e47240e3',\n",
    "'http://money.cnn.com/2011/08/24/technology/apple_after_hours_shares/index.htm',\n",
    "'http://money.cnn.com/2011/08/24/technology/steve_jobs_resigns/index.htm',\n",
    "'https://seekingalpha.com/article/289692-how-the-steve-jobs-news-impacts-apple-in-the-long-term',\n",
    "'http://money.cnn.com/2012/04/24/technology/apple-earnings/index.htm',\n",
    "'http://money.cnn.com/2012/04/10/technology/apple_market_cap/index.htm',\n",
    "'https://seekingalpha.com/article/527501-apples-delicious-quarterly-results',\n",
    "'https://seekingalpha.com/article/536451-what-apple-is-doing-right-to-create-long-term-value',\n",
    "'http://www.marketwatch.com/story/has-apple-lost-it-2012-10-10',\n",
    "'https://www.forbes.com/sites/darcytravlos/2012/10/04/the-problem-for-apple-and-its-stock-price/#5b1a86e252d7',\n",
    "'https://www.cnbc.com/id/49565882',\n",
    "'https://seekingalpha.com/article/990291-apple-still-calm-before-the-storm',\n",
    "'https://www.forbes.com/sites/briansolomon/2014/05/28/apple-brings-dr-dre-on-board-with-official-3-billion-beats-deal/',\n",
    "'https://www.forbes.com/sites/zackomalleygreenburg/2014/05/30/evaluating-apples-3-billion-beats-buy-wins-losses-and-draws/',\n",
    "'https://finance.yahoo.com/blogs/daily-ticker/apple-buying-beats-for--3-2-billion--smart-move-145319949.html',\n",
    "'https://www.forbes.com/sites/nathanvardi/2014/05/12/tim-cook-dr-dre-and-the-battle-for-apples-cash/#523dd2eb3dd3',\n",
    "'https://www.fool.com/investing/general/2014/06/16/heres-how-beats-electronics-will-benefit-apple-inc.aspx',\n",
    "'http://money.cnn.com/2014/08/19/investing/apple-stock-record-high/index.html',\n",
    "'https://www.fool.com/investing/general/2014/10/14/bose-vs-beats-the-nfl-tests-its-limits-with-its-he.aspx',\n",
    "'https://investorplace.com/2014/11/aapl-buy-apple-stock-2015/#.WbWvZsbMz-Y',\n",
    "'https://seekingalpha.com/article/2228023-the-real-reason-apple-is-acquiring-beats-for-3_2b-part-2',\n",
    "'http://money.cnn.com/2015/04/28/technology/apple-stock-all-time-high/index.html',\n",
    "'https://seekingalpha.com/article/3059826-apple-watch-success-not-limited-to-unit-sales',\n",
    "'http://markets.businessinsider.com/news/stocks/apple-stock-price-three-reasons-after-wwdc-2017-6-1002071211',\n",
    "'https://www.cnbc.com/2017/05/12/apples-next-iphone-will-break-the-1000-barrier-and-send-the-stock-soaring-goldman-says.html',\n",
    "'http://fortune.com/2017/02/01/apple-shares-analysts/',\n",
    "'https://investorplace.com/2017/02/why-apple-inc-aapl-stock-could-rise-to-150-upgrade/#.WbWygMbMz-Y',\n",
    "'https://investorplace.com/2017/03/apple-inc-aapl-stock-rise-more-than-15/#.WbWyuMbMz-Y',\n",
    "'https://www.thestreet.com/story/13336718/1/here-s-why-apple-aapl-stock-is-on-the-rise-today.html',\n",
    "'https://www.nytimes.com/2017/06/06/technology/apple-reinvent-itself.html',\n",
    "'https://www.apple.com/newsroom/2017/06/homepod-reinvents-music-in-the-home/',\n",
    "'https://www.apple.com/newsroom/2017/01/app-store-shatters-records-on-new-years-day/',\n",
    "'https://techcrunch.com/2017/12/11/apple-shazam-deal/',\n",
    "'https://www.apple.com/newsroom/2017/08/apples-next-us-data-center-will-be-built-in-iowa/',\n",
    "'https://www.apple.com/newsroom/2017/12/imac-pro-the-most-powerful-mac-ever-available-today/',\n",
    "'https://www.usatoday.com/story/tech/news/2017/04/14/apple-joins-driverless-car-test-applicants/100467124/',\n",
    "'https://www.forbes.com/sites/chrisversace/2017/06/06/apples-wwdc-2017-an-event-lacking-vision-from-a-company-without-a-visionary/#3928ddda1f7e',\n",
    "'https://www.cnbc.com/2017/07/06/samsung-q2-earnings-preview-profit-apple.html',\n",
    "'https://www.theguardian.com/technology/2017/apr/04/apple-planning-new-imac-mac-pro',\n",
    "'https://www.cnbc.com/2017/11/03/us-stocks-apple-earnings-jobs-report.html',\n",
    "'https://www.bloomberg.com/news/articles/2017-06-13/cook-says-apple-is-focusing-on-making-an-autonomous-car-system',\n",
    "'https://www.forbes.com/sites/amitchowdhry/2017/11/07/apple-confirms-ios-11-1-autocorrect-bug-and-suggests-this-workaround/#1d0494746b7a',\n",
    "'https://www.inc.com/bill-murphy-jr/apples-new-iphone-has-a-frustrating-bug-thats-driving-people-crazy.html',\n",
    "'https://techcrunch.com/2017/11/07/apple-pay-cash-launches-in-beta-today-letting-you-send-and-receive-cash-in-messages/',\n",
    "'https://www.nytimes.com/2017/09/06/business/apple-watch-red-sox-cheating.html',\n",
    "'https://www.theverge.com/2017/10/10/16454312/apple-steven-spielberg-amazing-stories-original-content-deal',\n",
    "'https://www.cnbc.com/2017/11/17/apple-just-delayed-its-homepod-smart-speaker-until-next-year.html',\n",
    "'https://www.apple.com/newsroom/2017/06/macos-high-sierra-delivers-advanced-technologies-for-storage-video-and-graphics/',\n",
    "'https://www.wired.com/2017/06/review-apple-ipad-pro-10-5-inch/',\n",
    "'https://www.reuters.com/article/us-china-fortunteforum-apple/apples-cook-optimistic-that-apps-pulled-in-china-will-be-back-idUSKBN1E00MI',\n",
    "'https://www.theguardian.com/technology/2017/jan/17/apple-ios-mac-app-store-prices-rise-25-per-cent-following-brexit',\n",
    "'https://www.theverge.com/2017/11/6/16614038/apple-samsung-slide-to-unlock-supreme-court-120-million',\n",
    "'https://www.theverge.com/2017/3/25/15056856/drake-more-life-apple-music-jimmy-iovine-streaming-numbers',\n",
    "'https://www.theverge.com/2017/4/4/15175916/apple-mac-pro-redesign-plans',\n",
    "'https://www.bloomberg.com/news/articles/2017-12-27/apple-ceo-cook-gets-74-bonus-boost-after-sales-profit-rebound',\n",
    "'https://www.theverge.com/2017/8/16/16155600/apple-original-tv-content-1-billion-investment',\n",
    "'https://www.cnbc.com/2017/12/21/apple-is-morgan-stanleys-top-tech-pick-for-2018.html',\n",
    "'https://www.gsmarena.com/apple_pay_arrives_in_denmark_finland_sweden_and_uae-news-27882.php',\n",
    "'https://standardoracle.com/2017/11/20/do-investors-want-to-take-risk-on-this-stock-apple-inc-aapl-2/',\n",
    "'https://hitechgazette.com/apple-delays-the-launch-of-homepod-till-early-2018/',\n",
    "'https://standardoracle.com/2017/09/18/stock-technical-performances-to-explore-apple-inc-aapl/',\n",
    "'http://www.ozarksfirst.com/news/apple-apologizes-for-slowing-down-old-iphones-cuts-price-of-replacement-batteries/890462649',\n",
    "'https://www.popsugar.com/news/Apple-Watch-LTE-Series-3-Details-44005001',\n",
    "'http://bgr.com/2017/11/13/iphone-x-design-case-screen-protectors-ugly/',\n",
    "'https://www.theguardian.com/business/2016/aug/30/apple-pay-back-taxes-eu-ruling-ireland-state-aid',\n",
    "'https://techcrunch.com/2016/06/13/apple-finally-opens-siri-to-third-party-developers/',\n",
    "'https://www.apple.com/newsroom/2016/06/apple-previews-ios-10-biggest-ios-release-ever/',\n",
    "'https://www.bloomberg.com/news/articles/2016-08-30/apple-ordered-to-pay-up-to-14-5-billion-in-eu-tax-crackdown',\n",
    "'https://qz.com/607396/google-has-officially-dethroned-apple-as-the-worlds-most-valuable-company/',\n",
    "'http://fortune.com/2016/05/16/apple-stores-india/',\n",
    "'https://www.theguardian.com/technology/2016/may/16/warren-buffett-apple-stocks-berkshire-hathaway',\n",
    "'https://www.wired.com/2015/06/apple-builds-content-business-news-app/',\n",
    "'https://www.forbes.com/sites/julianmitchell/2015/06/09/wwdc-2015-the-hidden-strategy-that-gives-apple-music-the-edge-over-its-competitors/#4a3834315464',\n",
    "'https://www.marketwatch.com/story/apples-stock-tends-to-decline-after-wwdc-keynote-2015-06-08',\n",
    "'http://money.cnn.com/2015/06/08/technology/tim-cook-women-apple/index.html',\n",
    "'http://www.businessinsider.com/apple-wwdc-analysts-reaction-2015-6',\n",
    "'http://money.cnn.com/2015/08/04/investing/apple-stock-down-iphone-china/index.html',\n",
    "'http://money.cnn.com/2015/08/24/technology/apple-china/index.html',\n",
    "'https://www.reuters.com/article/us-apple-stock/apple-shares-to-close-year-with-worst-performance-since-2008-idUSKBN0UE18A20151231',\n",
    "'https://amigobulls.com/articles/apple-stock-has-the-magical-elixir-of-innovation-run-out',\n",
    "'https://www.usatoday.com/story/money/markets/2015/12/15/apple-more-bad-news/77358628/',\n",
    "'https://www.thestreet.com/story/13250111/1/cramer--this-is-why-you-hold-apple-stock.html',\n",
    "'https://www.macrumors.com/2015/08/24/aapl-below-100-dow-jones-downturn/',\n",
    "'https://www.fool.com/investing/general/2015/07/25/theres-no-way-apple-inc-sold-over-4-million-apple.aspx',\n",
    "'https://www.cultofmac.com/316880/three-reasons-why-apple-will-be-the-worlds-first-trillion-dollar-company/',\n",
    "'https://techcrunch.com/2015/03/09/apple-shares-shift-in-wake-of-new-product-announcements/',\n",
    "'https://www.forbes.com/sites/briansolomon/2014/06/02/apple-shares-sink-after-wwdc-lacks-new-hardware/#1b8a1db450af',\n",
    "'http://money.cnn.com/2014/06/06/investing/apple-stock-split/index.html',\n",
    "'https://www.nasdaq.com/article/5-reasons-ill-never-sell-apple-stock-cm409836',\n",
    "'https://www.fool.com/investing/general/2014/10/06/apple-pay-vs-google-wallet-why-apple-inc-will-win.aspx',\n",
    "'http://abcnews.go.com/Business/apple-stock-aapl-crosses-500-twitter-norm-market/story?id=19958207',\n",
    "'https://finance.yahoo.com/blogs/the-exchange/carl-icahn-multibillion-dollar-tweet-boosts-apple-stock-205938760.html',\n",
    "'https://www.wired.com/2013/08/icahn-apple-tweet/',\n",
    "'http://money.cnn.com/2013/01/24/investing/apple-stock/index.html',\n",
    "'https://www.beaumontenterprise.com/news/article/Exxon-surpasses-Apple-as-most-valuable-company-4224406.php',\n",
    "'https://www.cnet.com/news/apple-no-longer-the-worlds-most-valuable-company/',\n",
    "'http://business.time.com/2013/10/04/two-years-after-steve-jobs-death-is-apple-a-different-company/',\n",
    "'http://buzz.money.cnn.com/2013/05/21/apple-tim-cook-congress/',\n",
    "'https://www.marketwatch.com/story/10-reasons-to-buy-apple-stock-now-2013-12-11',\n",
    "'https://www.forbes.com/sites/darcytravlos/2013/04/21/do-i-buy-apple-on-monday/#74dcfac620f6',\n",
    "'http://www.businessinsider.com/fidelity-sells-apple-stock-aapl-2013-6',\n",
    "'http://www.nytimes.com/2013/04/19/technology/after-apples-rise-a-bruising-fall.html',\n",
    "'https://www.thestreet.com/story/11815657/1/apple-just-like-a-stinking-penny-stock.html',\n",
    "'https://www.usatoday.com/story/money/columnist/krantz/2013/04/17/apple-stock-growth-fall/2057925/',\n",
    "'http://www.businessinsider.com/david-trainer-on-apples-stock-2013-5',\n",
    "'https://investorplace.com/2013/10/apple-stock-aapl-microsof-msft/#.WpyS4maZPjA',\n",
    "'https://www.barrons.com/articles/BL-FUNDSB-12145',\n",
    "'https://www.macrumors.com/2013/12/05/apples-stock-hits-new-52-week-high-as-roller-coaster-2013-nears-its-end/',\n",
    "'http://money.cnn.com/2013/12/30/technology/tim-cook-pay/index.html',\n",
    "'https://www.slashgear.com/apples-tim-cook-demands-bigger-stock-risk-and-costs-himself-4m-30310366/',\n",
    "'https://www.usatoday.com/story/money/markets/2013/04/18/apple-crash-steve-jobs/2094379/',\n",
    "'http://www.businessinsider.com/apple-stock-low-2013-4',\n",
    "'http://www.nytimes.com/2013/02/09/business/following-a-herd-of-bulls-on-apples-stock.html',\n",
    "'https://etfdailynews.com/2013/03/25/apple-inc-aapl-can-the-stock-be-great-again/',\n",
    "'https://www.fool.com/investing/general/2013/06/19/warren-buffetts-simple-advice-about-apple-stock.aspx',\n",
    "'https://www.marketwatch.com/story/apple-and-the-iwatch-the-next-frontier-2013-02-12',\n",
    "'https://www.fool.com/investing/general/2013/04/25/1-hidden-reason-why-apples-margins-are-falling.aspx',\n",
    "'http://www.businessinsider.com/a-simple-chart-that-explains-the-drop-in-apples-stock-2013-1',\n",
    "'https://www.theregister.co.uk/2013/10/08/billionaire_dumps_apple_stock_because_steve_jobs_was_really_awful_guy/',\n",
    "'https://www.fool.com/investing/general/2013/07/10/why-im-doubling-down-on-apple-stock.aspx',\n",
    "'http://appleinsider.com/articles/13/10/24/aapl-shares-regain-highs-from-early-2013-many-apple-analysts-targets-now-underwater'\n",
    "          ]\n",
    "URLListGE = [\n",
    "'https://investorplace.com/2017/07/general-electric-company-ge-stock-survive/#.WqTBGWaZPjA',\n",
    "'https://www.thestreet.com/story/14237190/1/get-out-of-general-electric-ahead-of-the-quarter.html',\n",
    "'https://finance.yahoo.com/news/ge-report-q2-earnings-pull-134501576.html',\n",
    "'https://investorplace.com/2017/01/general-electric-company-ge-stock-history-winning/#.WqTBI2aZPjA',\n",
    "'https://www.barrons.com/articles/general-electric-much-ado-about-nothing-1500472930',\n",
    "'https://money.usnews.com/investing/stock-market-news/articles/2017-11-15/general-electric-company-ge-stock-dividend',\n",
    "'https://www.thestreet.com/story/14395148/1/ge-ceo-shows-faith-in-stock-with-million-dollar-purchase.html',\n",
    "'https://www.reuters.com/article/us-ge-stocks-ceo/ge-shares-rise-after-ceo-flannery-buys-stock-idUSKBN1DH1VY',\n",
    "'https://investorplace.com/2017/07/general-electric-company-ge-stock-no-longer-worth/#.WqTBMGaZPjA',\n",
    "'https://investorplace.com/2017/04/general-electric-company-ge-stock-is-a-buy-at-last/#.WqTBM2aZPjA',\n",
    "'https://investorplace.com/2017/09/heres-when-youll-want-to-buy-general-electric-company-ge-stock/#.WqTBNGaZPjA',\n",
    "'https://seekingalpha.com/article/4060287-general-electric-36-percent-total-return-opportunity-just-materialized',\n",
    "'https://www.thestreet.com/story/14297502/1/ge-stock-suffers-as-jpmorgan-lowers-outlook-even-further.html',\n",
    "'https://www.cnbc.com/2017/09/07/jpmorgan-says-ges-fundamentals-are-worse-than-we-think.html',\n",
    "'https://investorplace.com/2017/08/general-electric-company-ge-stock-permanently-stuck-20s/#.WqTBQmaZPjA',\n",
    "'https://www.bizjournals.com/boston/news/2017/08/10/with-stock-buy-ges-new-ceo-puts-millions-behind-a.html',\n",
    "'https://www.timesunion.com/7day-business/article/GE-s-new-CEO-buys-2-7-million-in-stock-for-401-k-11750062.php',\n",
    "'https://investorplace.com/2017/05/general-electric-company-ge-stock-52-week-lows/#.WqTBTGaZPjA',\n",
    "'https://investorplace.com/2017/05/general-electric-company-ge-stock-tech-play/#.WqXltWaZPjA',\n",
    "'https://investorplace.com/2017/08/john-flannery-cant-budge-general-electric-company-ge-stock-with-words/#.WqXlt2aZPjA',\n",
    "'https://investorplace.com/2017/09/general-electric-corporation-ge-short/#.WqXluWaZPjA',\n",
    "'https://www.forbes.com/sites/investor/2017/09/14/why-now-is-a-good-time-to-buy-general-electric/#c19a84b53055',\n",
    "'https://investorplace.com/2017/06/general-electric-company-ge-stock-far-more-dangerous/#.WqXlvWaZPjA',\n",
    "'https://www.fool.com/investing/2017/06/16/its-time-for-general-electric-to-bet-big-on-solar.aspx',\n",
    "'https://investorplace.com/2017/06/general-electric-company-ge-stock-catalyst-missed/#.WqXlwWaZPjA',\n",
    "'https://www.thestreet.com/story/14419577/1/ge-stock-is-awful.html',\n",
    "'https://investorplace.com/2017/06/how-general-electric-company-ge-stock-can-return-25/#.WqXlymaZPjA',\n",
    "'https://investorplace.com/2017/12/ge-stock-nowhere-fast/#.WqXl0WaZPjA',\n",
    "'https://investorplace.com/2017/01/abandon-general-electric-company-ge-stock-after-q4-earnings/#.WqXl1WaZPjA',\n",
    "'https://investorplace.com/2017/02/general-electric-company-ge-stock-isnt-worth-3-2-yield/#.WqXl12aZPjA',\n",
    "'https://investorplace.com/2017/08/general-electric-company-ge-stock-dog-dow/#.WqXl22aZPjA',\n",
    "'https://investorplace.com/2017/05/its-time-general-electric-company-ge-stock-died/#.WqXl3maZPjA',\n",
    "'https://investorplace.com/2017/03/general-electric-company-ge-stock-powerhouse/#.WqXl4WaZPjA',\n",
    "'https://investorplace.com/2016/12/general-electric-company-ge-stock-isnt-boring-anymore/#.WqXl72aZPjA',\n",
    "'https://investorplace.com/2016/12/general-electric-company-ge-good-company-terrible-stock/#.WqXl82aZPjA',\n",
    "'https://investorplace.com/2016/11/general-electric-company-ge-stock-trumps-america/#.WqXl_2aZPjA',\n",
    "'https://investorplace.com/2016/10/general-electric-company-ge-stock-solid-long-term-buy/#.WqXmAmaZPjA',\n",
    "'https://www.fool.com/investing/2016/10/09/a-simple-reason-to-consider-buying-general-electri.aspx',\n",
    "'https://investorplace.com/2016/10/general-electric-ge-stock-better/#.WqXmCWaZPjA',\n",
    "'https://seekingalpha.com/article/4010675-general-electric-company-hit-60-share-within-year',\n",
    "'https://investorplace.com/2016/08/general-electric-stock-ge/#.WqXmE2aZPjA',\n",
    "'http://fortune.com/2016/04/22/general-electric-ge-stock-earnings/',\n",
    "'https://investorplace.com/2016/02/ge-stock-general-electric-dividends/#.WqXmH2aZPjA',\n",
    "'https://amigobulls.com/articles/is-general-electric-company-stock-a-buy-ahead-of-its-q2-earnings',\n",
    "'https://www.benzinga.com/trading-ideas/short-ideas/16/09/8432173/ge-stock-is-breaking-down',\n",
    "'https://www.thestreet.com/story/13613324/1/ge-stock-is-the-lsquo-chart-of-the-day-rsquo.html',\n",
    "'http://fortune.com/2016/05/12/general-electric-sell-jpmorgan/',\n",
    "'https://www.thestreet.com/story/13328925/1/don-t-let-go-of-ge-despite-21-billion-synchrony-swap-offer.html',\n",
    "'https://etfdailynews.com/2015/10/19/general-electric-company-ge-diversification-spurs-the-stock/',\n",
    "'https://www.fool.com/investing/general/2015/01/20/volatile-oil-prices-general-electric-earnings.aspx',\n",
    "'https://www.reuters.com/article/us-ge-valuation/general-electric-valuation-hits-lofty-levels-after-stock-surge-idUSKBN0TX2F020151214',\n",
    "'https://investorplace.com/2015/02/general-electric-stock-general-electric-companyge-stock/#.WqXmaGaZPjA',\n",
    "'https://www.cnbc.com/2015/10/05/cramer-peltz-could-double-ges-stock.html',\n",
    "'http://money.cnn.com/2015/08/25/investing/ge-mystery-stock-plunge-china/index.html',\n",
    "'https://www.thestreet.com/story/12890967/1/why-general-electric-ge-stock-is-down-today.html'\n",
    "]\n",
    "URLListTSLA = [\n",
    "'http://www.businessinsider.com/tesla-stock-price-elon-musk-short-sellers-profit-from-decline-2017-11',\n",
    "'http://fortune.com/2017/11/17/tesla-stock-roadster-semi-truck/',\n",
    "'https://www.cnbc.com/2017/08/03/wall-street-stunned-on-teslas-stock-surge-skeptical-on-model-3-ramp.html',\n",
    "'https://money.usnews.com/investing/stock-market-news/articles/2017-08-02/tesla-tsla-stock-earnings',\n",
    "'https://techcrunch.com/2017/07/05/tesla-drops-7-after-goldman-sachs-says-the-stock-is-worth-180/',\n",
    "'https://www.marketwatch.com/story/teslas-stock-tumbles-into-correction-territory-but-analysts-sees-new-highs-ahead-2017-07-05',\n",
    "'https://investorplace.com/2017/07/tesla-inc-tsla-stock-will-hit-400-as-model-3-hype-turns-reality/#.WqXzrGaZPjA',\n",
    "'https://www.cnbc.com/2017/02/23/tesla-stock-drops-after-earnings-but-analysts-see-new-highs-ahead.html',\n",
    "'http://www.latimes.com/business/autos/la-fi-hy-tesla-1q-production-20170403-story.html',\n",
    "'https://www.reuters.com/article/us-tesla-stocks/elon-musk-mocks-short-sellers-after-tesla-stock-surge-idUSKBN17522H',\n",
    "'https://www.marketwatch.com/story/tesla-surpasses-ford-to-become-no-2-us-car-maker-2017-04-03',\n",
    "'https://money.usnews.com/investing/stock-market-news/articles/2017-12-01/tesla-inc-tsla-stock',\n",
    "'https://www.fool.com/investing/2017/01/04/the-simple-reason-i-wont-buy-tesla-motors-stock.aspx',\n",
    "'http://fortune.com/2017/07/08/tesla-stock-delivery/',\n",
    "'https://www.forbes.com/sites/petercohan/2017/07/07/why-you-should-sell-tesla-on-weeks-9-billion-plunge/#44e0761ad5a6',\n",
    "'https://www.fool.com/investing/2017/07/06/why-tesla-inc-stock-is-down-again-today.aspx',\n",
    "'http://www.businessinsider.com/chart-explains-why-teslas-stock-price-insane-territory-2017-4',\n",
    "'http://fortune.com/2017/01/19/tesla-elon-musk-donald-trump-morgan-stanley/',\n",
    "'https://www.bloomberg.com/news/articles/2017-04-10/tesla-passes-gm-as-musk-s-carmaker-becomes-america-s-top-valued',\n",
    "'https://www.thestreet.com/story/14371576/1/tesla-stock-drops-amid-mixed-q3-model-3-bottlenecks.html',\n",
    "'https://www.reuters.com/article/us-tesla-results/tesla-stock-falls-on-model-3-delays-biggest-ever-quarterly-loss-idUSKBN1D15TP',\n",
    "'https://www.forbes.com/sites/chuckjones/2017/11/01/if-teslas-model-3-news-is-bad-look-out-below/#3880eab668b6',\n",
    "'https://investorplace.com/2017/10/tesla-inc-tsla-stock-500-per-share/#.WqXz_GaZPjA',\n",
    "'https://www.thestreet.com/story/14164811/1/tesla-s-stock-could-pop-a-quick-26-because-of-its-awesome-new-model-3.html',\n",
    "'https://electrek.co/2017/06/07/tesla-tsla-all-time-fortune-500-list/',\n",
    "'https://investorplace.com/2017/06/knock-down-tesla-inc-tsla-stock/#.WqX0DGaZPjA',\n",
    "'https://www.investopedia.com/news/tesla-stock-analysts-divided-after-low-model-3-deliveries/',\n",
    "'https://www.investors.com/news/technology/tesla-stock-falls-on-model-3-production-miss-due-to-bottlenecks/',\n",
    "'https://www.cnbc.com/2017/11/02/tesla-stock-best-performing-tech-name-in-next-5-years-analyst-says.html',\n",
    "'http://www.latimes.com/business/la-fi-hy-tesla-tax-subsidy-20171102-story.html',\n",
    "'https://www.fool.com/investing/2017/07/28/why-the-model-3-is-so-important-for-tesla-inc.aspx',\n",
    "'https://www.cnbc.com/2017/07/17/elon-musk-teslas-stock-price-is-higher-than-we-deserve-right-now.html',\n",
    "'https://www.cnbc.com/2017/02/09/tesla-stock-jumps-on-word-of-model-3-production-preparations.html',\n",
    "'https://www.marketwatch.com/story/tesla-shares-rise-as-first-model-3-rolls-off-production-line-2017-07-10',\n",
    "'http://www.businessinsider.com/tesla-stock-faces-challenges-after-semi-roadster-reveal-2017-11',\n",
    "'https://www.investopedia.com/news/how-tesla-stock-benefits-trump-administration-tsla-f/',\n",
    "'http://www.businessinsider.com/tesla-stock-model-3-2016-4',\n",
    "'https://www.investors.com/news/technology/tesla-stock-lifts-as-model-3-lines-stretch-how-many-reservations/',\n",
    "'https://www.usatoday.com/story/money/cars/2016/04/01/tesla-motors-model-3-pre-orders/82506056/',\n",
    "'https://www.reuters.com/article/us-tesla-model3/tesla-buoyed-by-strong-model-3-orders-may-need-more-cash-idUSKCN0WY3JK',\n",
    "'https://money.usnews.com/investing/articles/2016-04-01/the-model-3-may-be-tesla-motors-biggest-test-yet-tsla',\n",
    "'http://www.valuewalk.com/2016/10/tesla-motors-inc-tsla-no-energy/',\n",
    "'https://money.usnews.com/investing/articles/2016-10-20/all-tesla-motors-inc-tsla-cars-will-now-feature-self-driving-hardware',\n",
    "'https://www.thestreet.com/story/13626717/1/tesla-tsla-stock-falls-in-after-hours-trading-on-model-s-investigation.html',\n",
    "'https://www.usatoday.com/story/money/columnist/krantz/2016/04/04/ask-matt-how-high-can-tesla-stock-go/82528438/',\n",
    "'http://money.cnn.com/2016/02/10/news/companies/tesla-earnings/index.html',\n",
    "'http://www.businessinsider.com/tesla-earnings-q1-2016-2016-5',\n",
    "'https://www.cnbc.com/2016/06/22/solarcity-stock-soars-as-wall-street-reacts-to-tesla-motors-28-billion-offer.html',\n",
    "'http://www.businessinsider.com/tesla-made-offer-to-buy-solar-city-2016-6',\n",
    "'http://www.bbc.com/news/technology-36765823',\n",
    "'http://www.latimes.com/business/autos/la-fi-hy-musk-master-plan-20160714-snap-story.html',\n",
    "'http://time.com/money/4279284/tesla-stock-price-surge/',\n",
    "'http://money.cnn.com/2016/04/01/news/companies/tesla-model-3-stock-price/index.html',\n",
    "'https://investorplace.com/2016/11/tesla-motors-inc-tsla-stock-solar-roof-ipmedia/#.WqX0_GaZPjA',\n",
    "'http://fortune.com/2016/05/18/goldman-sachs-tesla-musk/',\n",
    "'https://www.forbes.com/sites/greatspeculations/2016/11/21/tesla-can-boost-its-stock-price-by-50-if-it-meets-its-production-targets/#5b95729d475a',\n",
    "'https://www.cnbc.com/2016/05/03/tesla-stock-is-not-a-buy-no-matter-what-earnings-say-commentary.html',\n",
    "'https://www.thestreet.com/story/13652719/1/mobileye-mbly-stock-continues-to-decline-on-tesla-split.html',\n",
    "'http://www.businessinsider.com/2017-tesla-predictions-2016-12',\n",
    "'https://www.profitconfidential.com/stock/tesla-motors-inc-tsla-stock-chart-bullish/',\n",
    "'http://fortune.com/2016/02/05/heres-whats-behind-teslas-long-slow-stock-slide/',\n",
    "'https://investorplace.com/2015/05/tesla-powerwall-buy-tsla-stock/#.WqX1WWaZPjA',\n",
    "'https://www.fool.com/investing/general/2015/08/01/teslas-referral-program-smart-move-or-sign-of-desp.aspx',\n",
    "'https://www.benzinga.com/analyst-ratings/analyst-color/15/08/5769540/adam-jonas-why-tesla-stock-is-worth-465',\n",
    "'https://www.bloomberg.com/news/articles/2015-08-17/morgan-stanley-massively-hikes-price-target-on-tesla-says-stock-could-almost-double',\n",
    "'http://money.cnn.com/2015/01/13/investing/tesla-stock-gm-bolt/index.html',\n",
    "'http://www.businessinsider.com/why-tesla-always-overpromises-and-underdelivers-2015-10',\n",
    "'https://www.recode.net/2015/10/20/11619780/tesla-stock-is-falling-fast-after-consumer-reports-rescinds',\n",
    "'https://www.engadget.com/2015/10/20/consumer-reports-tesla-model-s-reliability/',\n",
    "'http://www.businessinsider.com/report-consumer-reports-pulls-recommendation-for-tesla-model-s-2015-10',\n",
    "'https://qz.com/528863/tesla-shares-are-sinking-after-consumer-reports-raised-reliability-concerns-about-the-model-s/',\n",
    "'https://www.investopedia.com/articles/active-trading/040215/now-good-time-invest-tesla.asp',\n",
    "'https://www.forbes.com/sites/briansolomon/2015/03/30/elon-musk-boots-tesla-stock-with-major-new-product-tweet-tease/#b50bcec3f9a7',\n",
    "'http://money.cnn.com/2015/09/30/investing/tesla-model-x-suv-reviews-stock/index.html',\n",
    "'https://www.cnbc.com/2015/11/06/why-tesla-stock-may-grow-4x-billionaire-ron-baron.html',\n",
    "'http://www.businessinsider.com/morgan-stanley-tesla-price-target-model-x-2015-10',\n",
    "'https://mashable.com/2014/10/10/tesla-d-stock/#gDTi40d_buqu',\n",
    "'http://www.businessinsider.com/tesla-electric-cars-could-take-years-to-outpace-gas-2014-8',\n",
    "'http://money.cnn.com/2014/12/08/investing/tesla-stock-oil-prices/index.html',\n",
    "'https://www.huffingtonpost.com/2014/09/02/tesla-stock-price_n_5755608.html',\n",
    "'http://www.cbc.ca/news/business/tesla-stock-soars-after-elon-musk-pledges-100-000-cars-a-year-1.2725186',\n",
    "'https://www.huffingtonpost.com/2014/12/16/tesla-stock-price_n_6336412.html'\n",
    "]\n",
    "URLListAMZN = [\n",
    "'https://www.cnbc.com/2017/06/16/after-its-stock-pop-amazon-will-get-whole-foods-essentially-for-free.html',\n",
    "'https://www.theatlantic.com/technology/archive/2017/06/when-exactly-does-amazon-become-a-monopoly/530616/',\n",
    "'https://money.usnews.com/investing/stock-market-news/articles/2017-11-26/amazon-com-inc-amzn-stock-cyber-monday',\n",
    "'https://www.investors.com/research/ibd-industry-themes/amazon-shares-hit-record-high-as-cyber-monday-sales-kick-in/',\n",
    "'https://investorplace.com/2017/11/cyber-monday-sales-give-amazon-com-inc-stock-power/',\n",
    "'http://fortune.com/2017/07/10/amazon-prime-day-2017-stock-price/',\n",
    "'https://www.investors.com/news/technology/heres-what-you-need-to-know-about-amazons-prime-day-bonanza/',\n",
    "'https://www.theverge.com/2017/7/27/16052792/amazon-q2-2017-earnings-jeff-bezos-worlds-richest-person-title',\n",
    "'https://www.cnbc.com/2017/06/19/amazon-stock-jumps-to-new-all-time-high-of-1017.html',\n",
    "'https://www.thestreet.com/story/14323008/1/amazon-stock-gains-jpmorgan.html',\n",
    "'https://www.fool.com/investing/2017/09/28/amazon-doubles-down-on-alexa-with-new-devices.aspx',\n",
    "'http://markets.businessinsider.com/news/stocks/amazon-stock-price-is-rising-after-announcing-tons-of-new-hardware-2017-9-1002667999',\n",
    "'https://www.cnbc.com/2017/05/30/amazon-shares-hit-1000.html',\n",
    "'https://www.seattletimes.com/business/amazon/amazons-stock-tops-1000-for-the-first-time-2/',\n",
    "'https://www.usatoday.com/story/tech/news/2017/05/30/amazon-hits-1000-what-keep-2000/102316448/',\n",
    "'https://money.usnews.com/investing/stock-market-news/articles/2017-12-29/amazon-com-inc-amzn-trump',\n",
    "'http://www.newsweek.com/amazon-stocks-fall-5-bn-after-single-tweet-trump-651675',\n",
    "'http://www.businessinsider.com/amazon-q3-earnings-report-2017-10',\n",
    "'http://fortune.com/2017/10/27/amazon-microsoft-alphabet-google-intel-stock/',\n",
    "'https://www.cnbc.com/2017/10/27/tech-stocks-leaping-higher-after-blowout-earnings-from-amazon-alphabet-and-microsoft.html',\n",
    "'https://www.mercurynews.com/2017/10/27/us-stocks-are-mixed-as-tech-soars-retail-and-energy-sink/',\n",
    "'https://www.investors.com/news/nike-earnings-late-look-for-chatter-on-amazon-new-strategy/',\n",
    "'https://seekingalpha.com/article/4117920-amazons-q3-expensive-stock-still-worth-owning',\n",
    "'https://www.seattletimes.com/business/amazon/amazon-worth-more-than-500-billion-as-stock-continues-its-climb/',\n",
    "'https://www.geekwire.com/2017/amazon-stock-surges-13-earnings-report-making-jeff-bezos-worlds-richest/',\n",
    "'https://www.investors.com/news/technology/amazon-boasts-of-strong-holiday-sales-ahead-of-digital-day/',\n",
    "'https://www.investopedia.com/news/amazons-prime-day-too-big-24-hours-stock-rally-expected/',\n",
    "'https://www.nasdaq.com/article/will-amazon-amzn-stock-rally-ahead-of-prime-day-cm809581',\n",
    "'http://www.heraldnet.com/business/with-surging-amazon-stock-bezos-now-worth-more-than-100b/',\n",
    "'https://www.investors.com/news/technology/amazon-makes-big-middle-east-play-with-souq-com-acquisition/',\n",
    "'https://www.zacks.com/stock/news/273258/heres-the-reason-for-amazons-recent-stock-pullback',\n",
    "'http://markets.businessinsider.com/news/stocks/amazon-stock-price-faces-one-big-problem-for-food-delivery-and-its-why-it-bought-whole-foods-2017-8-1002276390',\n",
    "'https://www.cnbc.com/2016/07/11/amazon-prime-day-is-no-joke-for-investors-as-stock-soars.html',\n",
    "'https://www.investors.com/news/technology/amazon-introduces-amazon-go-retail-stores-no-checkout-no-lines/',\n",
    "'https://investorplace.com/2016/12/amazon-com-inc-amzn-stock-800-holiday/',\n",
    "'http://www.businessinsider.com/amazon-earnings-q1-2016-2016-4',\n",
    "'https://www.geekwire.com/2016/amazon-stock-nears-time-high-record-earnings-tech-giant-valued-310b/',\n",
    "'http://time.com/money/4312379/amazon-q1-earnings-investor-confidence/',\n",
    "'http://blogs.marketwatch.com/thetell/2016/04/28/amazon-earnings-expected-to-show-a-return-to-profit-live-blog/',\n",
    "'http://money.cnn.com/2016/10/27/technology/amazon-earnings/index.html',\n",
    "'https://www.marketwatch.com/story/amazons-stock-chart-warns-this-years-amazing-ride-may-be-over-2016-10-28',\n",
    "'https://techcrunch.com/2016/07/28/amazon-shatters-earnings-expectations/',\n",
    "'https://www.thestreet.com/story/13656806/1/amazon-com-amzn-stock-falls-in-after-hours-trading-despite-q2-earnings-beat.html',\n",
    "'https://www.investors.com/news/technology/amazon-poised-to-become-no-1-u-s-apparel-retailer-by-2017/',\n",
    "'https://www.investors.com/news/technology/amazon-calls-prime-day-biggest-day-ever-with-strong-global-sales/',\n",
    "'https://www.investors.com/news/technology/amazon-launches-youtube-competitor-on-prime-video-platform/',\n",
    "'http://money.cnn.com/2016/05/02/investing/amazon-stock-warren-buffett-jeff-bezos/index.html',\n",
    "'http://www.businessinsider.com/amazon-closes-record-high-stock-price-2016-9',\n",
    "'https://investorplace.com/2016/03/amazon-com-inc-amzn-stock-heading-lower/',\n",
    "'https://www.thestreet.com/story/13842305/1/amazon-com-amzn-stock-higher-price-target-raised-at-morgan-stanley-suntrust.html',\n",
    "'https://investorplace.com/2016/07/amzn-stock-amazon-com-inc-earnings/',\n",
    "'https://www.marketwatch.com/story/three-reasons-why-amazons-stock-may-be-unstoppable-2016-06-01',\n",
    "'http://www.businessinsider.com/amazon-greatest-strength-in-two-sentences-2016-10',\n",
    "'https://www.geekwire.com/2015/amazon-com-customers-not-happy-with-initial-prime-day-selection/',\n",
    "'http://fortune.com/2015/07/24/amazon-stock-surge/',\n",
    "'https://www.forbes.com/forbes/welcome/?toURL=https://www.forbes.com/sites/ryanmac/2015/07/23/amazon-second-quarter-earnings-2015/&refURL=https://www.google.com/&referrer=https://www.google.com/',\n",
    "'https://www.marketwatch.com/story/amazon-analysts-bullish-on-the-stock-after-big-earnings-beat-2015-07-24',\n",
    "'https://www.cnbc.com/2015/07/24/amazon-shares-surge-20-percent-market-cap-surpasses-wal-mart.html',\n",
    "'https://www.nytimes.com/2015/07/24/technology/amazon-earnings-q2.html',\n",
    "'https://www.geekwire.com/2015/amazon-stock-14-percent-investors-rally-around-fourth-quarter-profits/',\n",
    "'http://www.businessinsider.com/why-amazons-stock-is-going-nuts-2015-1',\n",
    "'http://money.cnn.com/2015/01/29/investing/amazon-earnings/index.html',\n",
    "'http://www.slate.com/blogs/moneybox/2015/01/29/amazon_tops_q4_earnings_expectations_wall_street_is_actually_happy_with.html',\n",
    "'https://www.seattletimes.com/business/amazon/amazon-shares-soar-after-company-posts-surprise-profit-6/',\n",
    "'https://www.washingtonpost.com/news/business/wp/2015/10/22/amazon-stock-is-surging-thanks-to-surprisingly-upbeat-earnings/?utm_term=.51fdd1c67771',\n",
    "'https://finance.yahoo.com/news/why-amazon-s-stock-is-really-flying--hint--it-s-not-the-cloud-144934941.html',\n",
    "'https://www.geekwire.com/2015/amazons-shares-up-50-this-year-as-stock-continues-to-soar-on-the-outlook-for-prime/',\n",
    "'https://www.marketwatch.com/story/amazon-crushes-black-friday-weekend-with-record-sales-of-its-own-devices-2015-12-01',\n",
    "'https://www.marketwatch.com/story/amazon-is-now-an-unstoppable-stock-2015-10-26',\n",
    "'https://www.marketwatch.com/story/analysts-are-talking-500-for-amazon-shares-2015-04-24',\n",
    "'https://www.thestreet.com/story/13382999/1/it-s-time-to-sell-your-amazon-com-stock-this-technical-chart-reveals.html',\n",
    "'https://www.theguardian.com/technology/2015/jul/23/amazon-stock-surge-makes-ceo-jeff-bezos-7bn-richer-in-45-minutes',\n",
    "'https://www.forbes.com/sites/petercohan/2015/12/31/3-reasons-amazon-stock-will-soar-in-2016/#114ffd507117',\n",
    "'https://www.huffingtonpost.com/2014/10/23/amazon-stock-tanks-sales_n_6037630.html',\n",
    "'https://www.forbes.com/sites/ryanmac/2014/10/23/amazon-third-quarter-2014-earnings/#28c07ee71031',\n",
    "'https://investorplace.com/2014/12/amazon-stock-amzn-timeline-2014/',\n",
    "'https://newrepublic.com/article/120464/amazons-revenue-growth-has-fallen-making-investors-nervous',\n",
    "'https://www.marketwatch.com/story/7-reasons-amazons-stock-is-going-nowhere-in-2014-2014-05-16',\n",
    "'https://www.fool.com/investing/high-growth/2014/04/27/amazoncom-continues-to-be-the-most-innovative-reta.aspx',\n",
    "'https://www.forbes.com/sites/greatspeculations/2013/01/30/amazons-third-party-sellers-drive-margins-and-stock-higher/#16e488f952b0',\n",
    "'https://www.wired.com/2013/01/amazon-stock-hits-all/'\n",
    "]\n",
    "URLListNVDA = [\n",
    "'https://cnafinance.com/nvidia-corporation-nvda-stock-falling-on-trade-war-news/18897',\n",
    "'https://investorplace.com/2018/03/go-long-the-nvidia-corporation-nvda-stock-dip-before-it-closes-and-with-certainty/',\n",
    "'https://investorplace.com/2018/03/nvidia-stock-has-30-percent-upside-from-current-levels/',\n",
    "'https://www.fool.com/investing/2018/03/19/10-reasons-to-buy-nvidia-stock-and-hold-on-for-the.aspx',\n",
    "'https://seekingalpha.com/article/4160331-nvidia-ai-will-drive-growth',\n",
    "'https://finance.yahoo.com/news/why-nvidia-corporation-nvda-5-090109792.html',\n",
    "'https://stocknewsjournal.com/2018/03/16/why-investors-remained-buoyant-on-nvidia-corporation-nvda-fireeye-inc-feye/',\n",
    "'http://productioninvestments.com/notable-eye-catching-stocks-nvidia-corporation-nasdaq-nvda/',\n",
    "'https://finance.yahoo.com/news/apos-why-nvidia-nvda-stock-155203981.html',\n",
    "'https://investorplace.com/2017/11/nvidia-corporation-nvda/',\n",
    "'https://www.nasdaq.com/article/buy-nvidia-nvda-stock-ahead-of-2nd-quarter-earnings-jefferies-cm829962',\n",
    "'https://investorplace.com/2017/10/nvidia-nvda-stock-trading-200/',\n",
    "'https://www.nasdaq.com/article/5-reasons-to-add-nvidia-nvda-stock-to-your-portfolio-now-cm833916',\n",
    "'https://finance.yahoo.com/news/buy-nvidia-nvda-stock-now-220810467.html',\n",
    "'https://investorplace.com/2017/07/should-you-chase-nvidia-corporation-nvda-stock-here/',\n",
    "'https://investorplace.com/2017/06/buy-nvidia-corporation-nvda-stock-if-youre-sane/',\n",
    "'https://investorplace.com/2017/05/nvidia-corporation-nvda-stock-will-hit-200-before-amd-hits-16/',\n",
    "'https://investorplace.com/2017/07/nvidia-corporation-nvda-stock-worth-buying-on-decline-toward-120/',\n",
    "'https://investorplace.com/2017/09/3-signs-that-nvidia-corporation-nvda-stock-is-oversold/',\n",
    "'https://www.thestreet.com/story/14324835/1/nvidia-stock-could-rally.html',\n",
    "'https://investorplace.com/2017/09/nvda-stock-still-king/',\n",
    "'https://investorplace.com/2017/06/nvidia-why-nvda-stock-is-heading-to-180/',\n",
    "'https://investorplace.com/2017/10/nvidia-corporation-nvda-stock-jumps-record-high/',\n",
    "'https://www.valuewalk.com/2017/05/nvidia-corporation-nvda-stock-tear-artificial-intelligence/',\n",
    "'https://investorplace.com/2017/07/nvidia-corporation-nvda-welcome-to-the-era-of-ai/',\n",
    "'https://investorplace.com/2017/10/nvda-stock-caution-signals/',\n",
    "'https://investorplace.com/2017/05/nvidia-corporation-nvda-stock-bubble-is-about-to-pop/',\n",
    "'https://www.nasdaq.com/article/why-nvidia-nvda-might-be-a-diamond-in-the-rough-cm842851',\n",
    "'https://amigobulls.com/articles/2017-06-16-nvidia-corporation-nvda-stock-is-still-a-great-buy-here-is-why',\n",
    "'https://investorplace.com/2017/02/dont-count-nvidia-corporation-nvda-stock-just-yet/',\n",
    "'https://investorplace.com/2017/07/3-reasons-nvidia-corporation-nvda-stock-bullish-case/',\n",
    "'https://investorplace.com/2017/08/is-nvidia-corporation-nvda-stock-in-buy-territory/',\n",
    "'https://investorplace.com/2017/05/why-nvidia-corporation-nvda-stock-will-hit-130/',\n",
    "'https://investorplace.com/2016/12/is-nvidia-corporation-nvda-stock-for-real/',\n",
    "'https://seekingalpha.com/article/3906826-nvidias-automotive-strategy-pays',\n",
    "'https://www.investopedia.com/news/nvidia-corporation-surges-after-q3-earnings-nvda/',\n",
    "'https://investorplace.com/2016/12/nvidia-stock-nvda-megatrends-power-more-gains/',\n",
    "'https://seekingalpha.com/article/4025086-nvidia-easily-double-tripling',\n",
    "'https://seekingalpha.com/article/4033450-nvidia-expect-strong-gains-2020',\n",
    "'https://marketrealist.com/2016/10/nvidias-stock-continues-outperform-market-peers',\n",
    "'https://finance.yahoo.com/news/why-nvidia-nvda-stock-might-124012807.html',\n",
    "'https://amigobulls.com/articles/why-you-should-buy-nvidia-corporation-nvda-stock-on-dips',\n",
    "'https://www.investopedia.com/stock-analysis/122916/heres-why-nvidia-dropped-and-why-not-worry-nvda.aspx',\n",
    "'https://247wallst.com/technology-3/2016/12/28/nvidia-is-americas-most-successful-company-of-2016/',\n",
    "'https://www.thestreet.com/story/13658102/1/nvidia-nvda-could-be-the-intel-of-artificial-intelligence-ritholtz-s-ceo-brown-tells-cnbc.html',\n",
    "'https://www.marketwatch.com/story/nvidia-stock-rockets-to-record-high-following-earnings-shocker-2016-11-11',\n",
    "'https://www.investopedia.com/stock-analysis/061615/3-reasons-nvidia-stock-could-rise-nvda.aspx',\n",
    "'https://investorplace.com/2015/08/nvidia-stock-nvda/',\n",
    "'http://www.businessinsider.com/r-nvidia-revenue-rises-45-percent-on-strong-demand-for-car-chips-2015-8',\n",
    "'https://www.investopedia.com/stock-analysis/040715/2-huge-growth-opportunities-nvidia-nvda.aspx',\n",
    "'https://www.fool.com/investing/general/2015/01/29/nvidia-wants-to-power-the-drone-market.aspx',\n",
    "'https://wccftech.com/nvidia-nasdaq-nvda-q3-fy16-financial-results/',\n",
    "'https://investorplace.com/2015/12/nvidia-buy-nvdas-breakout-bang-dollar/',\n",
    "'https://www.thestreet.com/story/13191279/1/pre-market-activity-shows-gains-for-nvidia-nvda.html',\n",
    "'https://seekingalpha.com/article/2288203-is-there-more-upside-left-in-nvidia',\n",
    "'https://www.fool.com/investing/general/2014/04/08/why-nvidia-will-continue-to-dominate-the-pc-gaming.aspx',\n",
    "'https://investorplace.com/2017/09/best-stocks-nvidia-corporations-nvda-stock-run/',\n",
    "'https://www.nasdaq.com/article/nvidia-corporation-nvda-looks-good-stock-adds-53-in-session-cm876332',\n",
    "'https://investorplace.com/2017/11/nvidia-corporation-nvda-stock-upside/',\n",
    "'https://investorplace.com/2017/11/why-nvidia-corporation-stock-is-unstoppable/',\n",
    "'https://investorplace.com/2017/03/nvidia-corporation-nvda-stock-downright-dangerous/',\n",
    "'https://www.benzinga.com/analyst-ratings/analyst-color/17/11/10805616/there-could-be-18-upside-left-in-nvidia-analyst-says',\n",
    "'https://amigobulls.com/articles/2017-05-26-more-reasons-to-buy-nvidia-corporation-nvda-stock',\n",
    "'https://investorplace.com/2017/06/has-nvidia-corporation-nvda-stock-finally-reached-sell-territory/',\n",
    "'https://www.marketwatch.com/story/nvidias-stock-rocked-after-analysts-say-its-time-to-sell-2017-02-23',\n",
    "'https://investorplace.com/2017/10/nvda-stock-200-share/',\n",
    "'https://investorplace.com/2017/03/nvidia-corporation-nvda-stock-wont-stop-growing/'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Main Procedure Block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def BusinessDay (strday, ticker):\n",
    "    import pandas as pd\n",
    "    StockHistory = pd.read_csv('StockHistoryData/'+ticker+'.csv')\n",
    "    lyst = list(StockHistory['timestamp'])\n",
    "    if (strday in lyst):\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collected data from NVDA for 4594 days!\n",
      "Article 1 , Output: 0\n",
      "Article 2 , Output: 0\n",
      "Article 3 , Output: 0\n",
      "Article 5 , Output: 0\n",
      "Article 6 , Output: 1\n",
      "Article 8 , Output: 0\n",
      "Article 9 , Output: 1\n",
      "Article 10 , Output: 0\n",
      "Article 11 , Output: 1\n",
      "Article 12 , Output: 1\n",
      "Article 13 , Output: 1\n",
      "Article 14 , Output: 1\n",
      "Article 15 , Output: 1\n",
      "Article 16 , Output: 1\n",
      "Article 17 , Output: 1\n",
      "Article 19 , Output: 1\n",
      "Article 20 , Output: 1\n",
      "Article 21 , Output: 1\n",
      "Article 22 , Output: 1\n",
      "Article 23 , Output: 1\n",
      "Article 24 , Output: 1\n",
      "Article 25 , Output: 0\n",
      "Article 26 , Output: 1\n",
      "Article 27 , Output: 1\n",
      "Article 28 , Output: 1\n",
      "Article 29 , Output: 1\n",
      "Article 30 , Output: 1\n",
      "Article 31 , Output: 1\n",
      "Article 32 , Output: 1\n",
      "Article 33 , Output: 1\n",
      "Article 35 , Output: 1\n",
      "Article 36 , Output: 0\n",
      "Article 40 , Output: 1\n",
      "Article 41 , Output: 0\n",
      "Article 42 , Output: 0\n",
      "Article 43 , Output: 0\n",
      "Article 44 , Output: 1\n",
      "Article 45 , Output: 1\n",
      "Article 46 , Output: 0\n",
      "Article 47 , Output: 1\n",
      "Article 48 , Output: 0\n",
      "Article 49 , Output: 0\n",
      "Article 50 , Output: 1\n",
      "Article 51 , Output: 0\n",
      "Article 52 , Output: 1\n",
      "Article 53 , Output: 0\n",
      "Article 55 , Output: 1\n",
      "Article 56 , Output: 1\n",
      "Article 57 , Output: 1\n",
      "Article 58 , Output: 1\n",
      "Article 59 , Output: 0\n",
      "Article 60 , Output: 0\n",
      "Article 62 , Output: 1\n",
      "Article 63 , Output: 1\n",
      "Article 64 , Output: 0\n",
      "Article 65 , Output: 1\n"
     ]
    }
   ],
   "source": [
    "stockname = \"AAPL\"\n",
    "StockHistory = CollectData(stockname)\n",
    "\n",
    "for i in range(0,len(URLListAAPL)-1):\n",
    "    url = URLListAAPL[i]\n",
    "    articletext, articledate = URLtoTextSummary(url)\n",
    "    \n",
    "    #Sentiment Scores\n",
    "    ListedText = articletext.split() #Turn into list\n",
    "    pos, neg = getSentiment(ListedText)\n",
    "    \n",
    "    #Filter System\n",
    "    if (articledate is None) or (isinstance(articledate, unicode)) or (len(StockHistory[StockHistory['timestamp'] == articledate.strftime('%Y-%m-%d')].index.tolist()) == 0): #If no date or article written on weekend:\n",
    "        continue #Forget this article, move on\n",
    "    \n",
    "    #Days Old\n",
    "    daysOld = randint(0,5)\n",
    "    today = articledate + timedelta(days=daysOld)\n",
    "    strtoday = today.strftime('%Y-%m-%d')\n",
    "    while BusinessDay(strtoday, stockname) == False:\n",
    "        today = today - timedelta(days=1)\n",
    "        strtoday = today.strftime('%Y-%m-%d')\n",
    "    daysOld = (today - articledate).days\n",
    "    \n",
    "    #Prices\n",
    "    a = SearchCSVForDate(strtoday, stockname)\n",
    "    pricetoday = StockHistory.get_value(a,\"adjusted_close\")\n",
    "    priceonedayago = StockHistory.get_value(a+1,\"adjusted_close\")\n",
    "    pricetwodaysago = StockHistory.get_value(a+2,\"adjusted_close\")\n",
    "    pricethreedaysago = StockHistory.get_value(a+3,\"adjusted_close\")\n",
    "    pricefourdaysago = StockHistory.get_value(a+4,\"adjusted_close\")\n",
    "    pricefivedaysago = StockHistory.get_value(a+5,\"adjusted_close\")\n",
    "    pricesixdaysago = StockHistory.get_value(a+6,\"adjusted_close\")\n",
    "    pricesevendaysago = StockHistory.get_value(a+7,\"adjusted_close\")\n",
    "    \n",
    "    #Magnitude\n",
    "    mag1 = (pricetoday-priceonedayago)/priceonedayago\n",
    "    mag2 = (pricetoday-pricetwodaysago)/pricetwodaysago\n",
    "    mag3 = (pricetoday-pricethreedaysago)/pricethreedaysago\n",
    "    mag4 = (pricetoday-pricefourdaysago)/pricefourdaysago\n",
    "    mag5 = (pricetoday-pricefivedaysago)/pricefivedaysago\n",
    "    mag6 = (pricetoday-pricesixdaysago)/pricesixdaysago\n",
    "    mag7 = (pricetoday-pricesevendaysago)/pricesevendaysago\n",
    "    \n",
    "    #Regression\n",
    "    reg1 = getSlope(today, 5, 0) #5 Days in Past\n",
    "    reg2 = getSlope(today, 10, 0) #10 Days in Past\n",
    "    reg3 = getSlope(today, 15, 0) #15 Days in Past\n",
    "    reg4 = getSlope(today, 20, 0) #20 Days in Past\n",
    "    \n",
    "    #Day Of The Week\n",
    "    dayoftheweek = today.weekday() #0=Monday, 6=Sunday\n",
    "    \n",
    "    #Calculate Slope\n",
    "    slope = getSlope(today, 5, 1)\n",
    "    if (slope >= 0):\n",
    "        slope = 1\n",
    "    else:\n",
    "        slope = 0\n",
    "    \n",
    "    #Log Data\n",
    "    print(\"Article \" + str(i) + \" , Output: \" + str(slope))\n",
    "    with open('TrainingData.csv', 'a') as csvfile: #wb\n",
    "        filewriter = csv.writer(csvfile, delimiter=',',\n",
    "                            quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "        filewriter.writerow([pos, neg, daysOld, mag1, mag2, mag3, mag4, mag5, mag6, mag7,\n",
    "                             reg1, reg2, reg3, reg4, dayoftheweek, slope])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Organize Data w/ 70% Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Instances: 348\n",
      "('X Train: ', (243, 15))\n",
      "('X Test : ', (105, 15))\n",
      "('Y Train: ', (243, 1))\n",
      "('Y Test : ', (105, 1))\n"
     ]
    }
   ],
   "source": [
    "df_raw = read_csv('TrainingDataYesterday.csv').sample(frac=1)\n",
    "df = (df_raw - df_raw.mean()) / (df_raw.max() - df_raw.min()) #Normalization (rescaling from -1 to 1)\n",
    "df.iloc[:,15] = df_raw.iloc[:,15].values #Replace Normalized Y with Original 0-1 Y\n",
    "\n",
    "NumRows = df.shape[0]\n",
    "split = 0.7\n",
    "print(\"Number of Instances: \" + str(NumRows))\n",
    "X_train = df.iloc[0:int(NumRows*split),[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]].as_matrix()\n",
    "X_test = df.iloc[int(NumRows*split):NumRows,[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]].as_matrix()\n",
    "\n",
    "Y_train = df.iloc[0:int(NumRows*split),[15]].as_matrix()\n",
    "Y_test = df.iloc[int(NumRows*split):NumRows,[15]].as_matrix()\n",
    "\n",
    "print(\"X Train: \" , X_train.shape)\n",
    "print(\"X Test : \" , X_test.shape)\n",
    "print(\"Y Train: \" , Y_train.shape)\n",
    "print(\"Y Test : \" , Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create, Compile, & Fit Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ethanshaotran/anaconda/lib/python2.7/site-packages/ipykernel_launcher.py:3: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(15, activation=\"relu\", kernel_initializer=\"uniform\", input_dim=15)`\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/Users/ethanshaotran/anaconda/lib/python2.7/site-packages/ipykernel_launcher.py:6: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(30, activation=\"relu\", kernel_initializer=\"uniform\")`\n",
      "  \n",
      "/Users/ethanshaotran/anaconda/lib/python2.7/site-packages/ipykernel_launcher.py:9: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(30, activation=\"relu\", kernel_initializer=\"uniform\")`\n",
      "  if __name__ == '__main__':\n",
      "/Users/ethanshaotran/anaconda/lib/python2.7/site-packages/ipykernel_launcher.py:12: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(4, activation=\"relu\", kernel_initializer=\"uniform\")`\n",
      "  if sys.path[0] == '':\n",
      "/Users/ethanshaotran/anaconda/lib/python2.7/site-packages/ipykernel_launcher.py:14: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1, activation=\"sigmoid\", kernel_initializer=\"uniform\")`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 243 samples, validate on 105 samples\n",
      "Epoch 1/100\n",
      "243/243 [==============================] - 0s - loss: 0.6927 - acc: 0.5021 - val_loss: 0.6908 - val_acc: 0.6571\n",
      "Epoch 2/100\n",
      "243/243 [==============================] - 0s - loss: 0.6851 - acc: 0.6049 - val_loss: 0.6903 - val_acc: 0.6571\n",
      "Epoch 3/100\n",
      "243/243 [==============================] - 0s - loss: 0.6808 - acc: 0.5802 - val_loss: 0.6882 - val_acc: 0.6571\n",
      "Epoch 4/100\n",
      "243/243 [==============================] - 0s - loss: 0.6712 - acc: 0.6214 - val_loss: 0.6868 - val_acc: 0.6571\n",
      "Epoch 5/100\n",
      "243/243 [==============================] - 0s - loss: 0.6623 - acc: 0.6543 - val_loss: 0.6847 - val_acc: 0.6571\n",
      "Epoch 6/100\n",
      "243/243 [==============================] - 0s - loss: 0.6557 - acc: 0.6543 - val_loss: 0.6817 - val_acc: 0.6571\n",
      "Epoch 7/100\n",
      "243/243 [==============================] - 0s - loss: 0.6290 - acc: 0.6872 - val_loss: 0.6787 - val_acc: 0.6571\n",
      "Epoch 8/100\n",
      "243/243 [==============================] - ETA: 0s - loss: 0.5774 - acc: 0.888 - 0s - loss: 0.6303 - acc: 0.6584 - val_loss: 0.6733 - val_acc: 0.6571\n",
      "Epoch 9/100\n",
      "243/243 [==============================] - 0s - loss: 0.6283 - acc: 0.6420 - val_loss: 0.6706 - val_acc: 0.6571\n",
      "Epoch 10/100\n",
      "243/243 [==============================] - 0s - loss: 0.6094 - acc: 0.6914 - val_loss: 0.6614 - val_acc: 0.6571\n",
      "Epoch 11/100\n",
      "243/243 [==============================] - 0s - loss: 0.6015 - acc: 0.6667 - val_loss: 0.6564 - val_acc: 0.6667\n",
      "Epoch 12/100\n",
      "243/243 [==============================] - 0s - loss: 0.6009 - acc: 0.6749 - val_loss: 0.6519 - val_acc: 0.6857\n",
      "Epoch 13/100\n",
      "243/243 [==============================] - 0s - loss: 0.5833 - acc: 0.7202 - val_loss: 0.6367 - val_acc: 0.7048\n",
      "Epoch 14/100\n",
      "243/243 [==============================] - 0s - loss: 0.5906 - acc: 0.6708 - val_loss: 0.6248 - val_acc: 0.7238\n",
      "Epoch 15/100\n",
      "243/243 [==============================] - 0s - loss: 0.5953 - acc: 0.6831 - val_loss: 0.6176 - val_acc: 0.6952\n",
      "Epoch 16/100\n",
      "243/243 [==============================] - 0s - loss: 0.6027 - acc: 0.6708 - val_loss: 0.6086 - val_acc: 0.7238\n",
      "Epoch 17/100\n",
      "243/243 [==============================] - 0s - loss: 0.5820 - acc: 0.6955 - val_loss: 0.5935 - val_acc: 0.7619\n",
      "Epoch 18/100\n",
      "243/243 [==============================] - 0s - loss: 0.5910 - acc: 0.7160 - val_loss: 0.5845 - val_acc: 0.7143\n",
      "Epoch 19/100\n",
      "243/243 [==============================] - 0s - loss: 0.5919 - acc: 0.6914 - val_loss: 0.5799 - val_acc: 0.7238\n",
      "Epoch 20/100\n",
      "243/243 [==============================] - 0s - loss: 0.5949 - acc: 0.6831 - val_loss: 0.5805 - val_acc: 0.7333\n",
      "Epoch 21/100\n",
      "243/243 [==============================] - 0s - loss: 0.5996 - acc: 0.6749 - val_loss: 0.5762 - val_acc: 0.7238\n",
      "Epoch 22/100\n",
      "243/243 [==============================] - 0s - loss: 0.5751 - acc: 0.6667 - val_loss: 0.5876 - val_acc: 0.7238\n",
      "Epoch 23/100\n",
      "243/243 [==============================] - 0s - loss: 0.6048 - acc: 0.7037 - val_loss: 0.5743 - val_acc: 0.7048\n",
      "Epoch 24/100\n",
      "243/243 [==============================] - 0s - loss: 0.5641 - acc: 0.7160 - val_loss: 0.5694 - val_acc: 0.7143\n",
      "Epoch 25/100\n",
      "243/243 [==============================] - 0s - loss: 0.5560 - acc: 0.7078 - val_loss: 0.5656 - val_acc: 0.7238\n",
      "Epoch 26/100\n",
      "243/243 [==============================] - 0s - loss: 0.6009 - acc: 0.6749 - val_loss: 0.5754 - val_acc: 0.6952\n",
      "Epoch 27/100\n",
      "243/243 [==============================] - 0s - loss: 0.5797 - acc: 0.6584 - val_loss: 0.5621 - val_acc: 0.7333\n",
      "Epoch 28/100\n",
      "243/243 [==============================] - 0s - loss: 0.5796 - acc: 0.7037 - val_loss: 0.5743 - val_acc: 0.7143\n",
      "Epoch 29/100\n",
      "243/243 [==============================] - 0s - loss: 0.5697 - acc: 0.6914 - val_loss: 0.5730 - val_acc: 0.7143\n",
      "Epoch 30/100\n",
      "243/243 [==============================] - 0s - loss: 0.5574 - acc: 0.6914 - val_loss: 0.5822 - val_acc: 0.7143\n",
      "Epoch 31/100\n",
      "243/243 [==============================] - 0s - loss: 0.5580 - acc: 0.7284 - val_loss: 0.5821 - val_acc: 0.6762\n",
      "Epoch 32/100\n",
      "243/243 [==============================] - 0s - loss: 0.5885 - acc: 0.6790 - val_loss: 0.5744 - val_acc: 0.6952\n",
      "Epoch 33/100\n",
      "243/243 [==============================] - 0s - loss: 0.5649 - acc: 0.7160 - val_loss: 0.5678 - val_acc: 0.7143\n",
      "Epoch 34/100\n",
      "243/243 [==============================] - 0s - loss: 0.5595 - acc: 0.7160 - val_loss: 0.5735 - val_acc: 0.6476\n",
      "Epoch 35/100\n",
      "243/243 [==============================] - 0s - loss: 0.5700 - acc: 0.7202 - val_loss: 0.5657 - val_acc: 0.6762\n",
      "Epoch 36/100\n",
      "243/243 [==============================] - 0s - loss: 0.5797 - acc: 0.6955 - val_loss: 0.5649 - val_acc: 0.6667\n",
      "Epoch 37/100\n",
      "243/243 [==============================] - 0s - loss: 0.5512 - acc: 0.7202 - val_loss: 0.5556 - val_acc: 0.6857\n",
      "Epoch 38/100\n",
      "243/243 [==============================] - 0s - loss: 0.5806 - acc: 0.6749 - val_loss: 0.5534 - val_acc: 0.6762\n",
      "Epoch 39/100\n",
      "243/243 [==============================] - 0s - loss: 0.5661 - acc: 0.7202 - val_loss: 0.5548 - val_acc: 0.7143\n",
      "Epoch 40/100\n",
      "243/243 [==============================] - 0s - loss: 0.5837 - acc: 0.6790 - val_loss: 0.5739 - val_acc: 0.6762\n",
      "Epoch 41/100\n",
      "243/243 [==============================] - 0s - loss: 0.5562 - acc: 0.7243 - val_loss: 0.5645 - val_acc: 0.6667\n",
      "Epoch 42/100\n",
      "243/243 [==============================] - 0s - loss: 0.6124 - acc: 0.6626 - val_loss: 0.5605 - val_acc: 0.6762\n",
      "Epoch 43/100\n",
      "243/243 [==============================] - 0s - loss: 0.5713 - acc: 0.6996 - val_loss: 0.5756 - val_acc: 0.6667\n",
      "Epoch 44/100\n",
      "243/243 [==============================] - 0s - loss: 0.5665 - acc: 0.7078 - val_loss: 0.5659 - val_acc: 0.6762\n",
      "Epoch 45/100\n",
      "243/243 [==============================] - 0s - loss: 0.5420 - acc: 0.7243 - val_loss: 0.5690 - val_acc: 0.6762\n",
      "Epoch 46/100\n",
      "243/243 [==============================] - 0s - loss: 0.5473 - acc: 0.6914 - val_loss: 0.5691 - val_acc: 0.6857\n",
      "Epoch 47/100\n",
      "243/243 [==============================] - 0s - loss: 0.5784 - acc: 0.6914 - val_loss: 0.5836 - val_acc: 0.6476\n",
      "Epoch 48/100\n",
      "243/243 [==============================] - 0s - loss: 0.5586 - acc: 0.6872 - val_loss: 0.5749 - val_acc: 0.6381\n",
      "Epoch 49/100\n",
      "243/243 [==============================] - 0s - loss: 0.5512 - acc: 0.7202 - val_loss: 0.6055 - val_acc: 0.6286\n",
      "Epoch 50/100\n",
      "243/243 [==============================] - 0s - loss: 0.5153 - acc: 0.7449 - val_loss: 0.5997 - val_acc: 0.6476\n",
      "Epoch 51/100\n",
      "243/243 [==============================] - 0s - loss: 0.5749 - acc: 0.7160 - val_loss: 0.5967 - val_acc: 0.6571\n",
      "Epoch 52/100\n",
      "243/243 [==============================] - 0s - loss: 0.5921 - acc: 0.7119 - val_loss: 0.5683 - val_acc: 0.6762\n",
      "Epoch 53/100\n",
      "243/243 [==============================] - 0s - loss: 0.5732 - acc: 0.7160 - val_loss: 0.5597 - val_acc: 0.6857\n",
      "Epoch 54/100\n",
      "243/243 [==============================] - 0s - loss: 0.5690 - acc: 0.7078 - val_loss: 0.5605 - val_acc: 0.7143\n",
      "Epoch 55/100\n",
      "243/243 [==============================] - 0s - loss: 0.5439 - acc: 0.7160 - val_loss: 0.6009 - val_acc: 0.6667\n",
      "Epoch 56/100\n",
      "243/243 [==============================] - 0s - loss: 0.5705 - acc: 0.6831 - val_loss: 0.5585 - val_acc: 0.7143\n",
      "Epoch 57/100\n",
      "243/243 [==============================] - 0s - loss: 0.5618 - acc: 0.7366 - val_loss: 0.5605 - val_acc: 0.6952\n",
      "Epoch 58/100\n",
      "243/243 [==============================] - 0s - loss: 0.5494 - acc: 0.7160 - val_loss: 0.5789 - val_acc: 0.6857\n",
      "Epoch 59/100\n",
      "243/243 [==============================] - 0s - loss: 0.5555 - acc: 0.7202 - val_loss: 0.5709 - val_acc: 0.7048\n",
      "Epoch 60/100\n",
      "243/243 [==============================] - 0s - loss: 0.5459 - acc: 0.6872 - val_loss: 0.5785 - val_acc: 0.6952\n",
      "Epoch 61/100\n",
      "243/243 [==============================] - 0s - loss: 0.5612 - acc: 0.7037 - val_loss: 0.5693 - val_acc: 0.7143\n",
      "Epoch 62/100\n",
      "243/243 [==============================] - 0s - loss: 0.5657 - acc: 0.7119 - val_loss: 0.5715 - val_acc: 0.7238\n",
      "Epoch 63/100\n",
      "243/243 [==============================] - 0s - loss: 0.5796 - acc: 0.7078 - val_loss: 0.5803 - val_acc: 0.7048\n",
      "Epoch 64/100\n",
      "243/243 [==============================] - 0s - loss: 0.5644 - acc: 0.7037 - val_loss: 0.5692 - val_acc: 0.6952\n",
      "Epoch 65/100\n",
      "243/243 [==============================] - 0s - loss: 0.5345 - acc: 0.7366 - val_loss: 0.6008 - val_acc: 0.6667\n",
      "Epoch 66/100\n",
      "243/243 [==============================] - 0s - loss: 0.5512 - acc: 0.7078 - val_loss: 0.5681 - val_acc: 0.6667\n",
      "Epoch 67/100\n",
      "243/243 [==============================] - 0s - loss: 0.5555 - acc: 0.7284 - val_loss: 0.5705 - val_acc: 0.7143\n",
      "Epoch 68/100\n",
      "243/243 [==============================] - 0s - loss: 0.5514 - acc: 0.7119 - val_loss: 0.6058 - val_acc: 0.6952\n",
      "Epoch 69/100\n",
      "243/243 [==============================] - 0s - loss: 0.5579 - acc: 0.6996 - val_loss: 0.5929 - val_acc: 0.6762\n",
      "Epoch 70/100\n",
      "243/243 [==============================] - 0s - loss: 0.5618 - acc: 0.7160 - val_loss: 0.6041 - val_acc: 0.6381\n",
      "Epoch 71/100\n",
      "243/243 [==============================] - 0s - loss: 0.5640 - acc: 0.7078 - val_loss: 0.5968 - val_acc: 0.6476\n",
      "Epoch 72/100\n",
      "243/243 [==============================] - 0s - loss: 0.5618 - acc: 0.6996 - val_loss: 0.6379 - val_acc: 0.6286\n",
      "Epoch 73/100\n",
      "243/243 [==============================] - 0s - loss: 0.5780 - acc: 0.7119 - val_loss: 0.6291 - val_acc: 0.6571\n",
      "Epoch 74/100\n",
      "243/243 [==============================] - 0s - loss: 0.5399 - acc: 0.7078 - val_loss: 0.6051 - val_acc: 0.6476\n",
      "Epoch 75/100\n",
      "243/243 [==============================] - 0s - loss: 0.5883 - acc: 0.7037 - val_loss: 0.6175 - val_acc: 0.6667\n",
      "Epoch 76/100\n",
      "243/243 [==============================] - 0s - loss: 0.4988 - acc: 0.7737 - val_loss: 0.6211 - val_acc: 0.6762\n",
      "Epoch 77/100\n",
      "243/243 [==============================] - 0s - loss: 0.5274 - acc: 0.7325 - val_loss: 0.6271 - val_acc: 0.6667\n",
      "Epoch 78/100\n",
      "243/243 [==============================] - 0s - loss: 0.5246 - acc: 0.7613 - val_loss: 0.6564 - val_acc: 0.6571\n",
      "Epoch 79/100\n",
      "243/243 [==============================] - 0s - loss: 0.5906 - acc: 0.6667 - val_loss: 0.6060 - val_acc: 0.6667\n",
      "Epoch 80/100\n",
      "243/243 [==============================] - 0s - loss: 0.5498 - acc: 0.7119 - val_loss: 0.5985 - val_acc: 0.6762\n",
      "Epoch 81/100\n",
      "243/243 [==============================] - 0s - loss: 0.5905 - acc: 0.6749 - val_loss: 0.6061 - val_acc: 0.6571\n",
      "Epoch 82/100\n",
      "243/243 [==============================] - 0s - loss: 0.5420 - acc: 0.7202 - val_loss: 0.6458 - val_acc: 0.6571\n",
      "Epoch 83/100\n",
      "243/243 [==============================] - 0s - loss: 0.5413 - acc: 0.7366 - val_loss: 0.6161 - val_acc: 0.6667\n",
      "Epoch 84/100\n",
      "243/243 [==============================] - 0s - loss: 0.5673 - acc: 0.7202 - val_loss: 0.6142 - val_acc: 0.6762\n",
      "Epoch 85/100\n",
      "243/243 [==============================] - 0s - loss: 0.5351 - acc: 0.6872 - val_loss: 0.6172 - val_acc: 0.6952\n",
      "Epoch 86/100\n",
      "243/243 [==============================] - 0s - loss: 0.4962 - acc: 0.7654 - val_loss: 0.5936 - val_acc: 0.6762\n",
      "Epoch 87/100\n",
      "243/243 [==============================] - 0s - loss: 0.5334 - acc: 0.7202 - val_loss: 0.6377 - val_acc: 0.6571\n",
      "Epoch 88/100\n",
      "243/243 [==============================] - 0s - loss: 0.5007 - acc: 0.7407 - val_loss: 0.6572 - val_acc: 0.6476\n",
      "Epoch 89/100\n",
      "243/243 [==============================] - 0s - loss: 0.5224 - acc: 0.7531 - val_loss: 0.6366 - val_acc: 0.6381\n",
      "Epoch 90/100\n",
      "243/243 [==============================] - 0s - loss: 0.5500 - acc: 0.7202 - val_loss: 0.6017 - val_acc: 0.6381\n",
      "Epoch 91/100\n",
      "243/243 [==============================] - 0s - loss: 0.5600 - acc: 0.6708 - val_loss: 0.5989 - val_acc: 0.6476\n",
      "Epoch 92/100\n",
      "243/243 [==============================] - 0s - loss: 0.5294 - acc: 0.7572 - val_loss: 0.6122 - val_acc: 0.6286\n",
      "Epoch 93/100\n",
      "243/243 [==============================] - 0s - loss: 0.5348 - acc: 0.7284 - val_loss: 0.6336 - val_acc: 0.6476\n",
      "Epoch 94/100\n",
      "243/243 [==============================] - 0s - loss: 0.5595 - acc: 0.7037 - val_loss: 0.6210 - val_acc: 0.6381\n",
      "Epoch 95/100\n",
      "243/243 [==============================] - 0s - loss: 0.4984 - acc: 0.7449 - val_loss: 0.6374 - val_acc: 0.6476\n",
      "Epoch 96/100\n",
      "243/243 [==============================] - 0s - loss: 0.5786 - acc: 0.6955 - val_loss: 0.6239 - val_acc: 0.6381\n",
      "Epoch 97/100\n",
      "243/243 [==============================] - 0s - loss: 0.5438 - acc: 0.7119 - val_loss: 0.5940 - val_acc: 0.6667\n",
      "Epoch 98/100\n",
      "243/243 [==============================] - 0s - loss: 0.5727 - acc: 0.7325 - val_loss: 0.5625 - val_acc: 0.6857\n",
      "Epoch 99/100\n",
      "243/243 [==============================] - 0s - loss: 0.4960 - acc: 0.7654 - val_loss: 0.6108 - val_acc: 0.6571\n",
      "Epoch 100/100\n",
      "243/243 [==============================] - 0s - loss: 0.5547 - acc: 0.7325 - val_loss: 0.6099 - val_acc: 0.6857\n",
      "Saved model to disk\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dropout(0.2, input_shape=(15,)))\n",
    "model.add(Dense(15, input_dim=15, init= 'uniform' , activation= 'relu' )) #4 4 3 1\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(30, init= 'uniform' , activation= 'relu' ))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(30, init= 'uniform' , activation= 'relu' ))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(4, init= 'uniform' , activation= 'relu' ))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(1, init= 'uniform' , activation= 'sigmoid' ))\n",
    "\n",
    "model.compile(loss= 'binary_crossentropy' , optimizer= 'adam' , metrics=[ 'accuracy' ])\n",
    "\n",
    "model.fit(X_train, Y_train, validation_data=(X_test,Y_test), nb_epoch=100, batch_size=9)\n",
    "\n",
    "\n",
    "model_json = model.to_json()\n",
    "with open(\"SavedModel/new2model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "model.save_weights(\"SavedModel/new2model.h5\")\n",
    "print(\"Saved model to disk\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Real-Life Usage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ethan's Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model from disk\n",
      "Collected data from AAPL for 4594 days!\n",
      "Found 25 Articles for Stock AAPL\n",
      "22 instances found\n",
      "[[ 0.90652955]\n",
      " [ 0.92966026]\n",
      " [ 0.98842245]\n",
      " [ 0.8955251 ]\n",
      " [ 0.96814555]\n",
      " [ 0.97546041]\n",
      " [ 0.86617571]\n",
      " [ 0.99619615]\n",
      " [ 0.68044579]\n",
      " [ 0.39871749]\n",
      " [ 0.9554168 ]\n",
      " [ 0.77140462]\n",
      " [ 0.9823525 ]\n",
      " [ 0.01395435]\n",
      " [ 0.99169534]\n",
      " [ 0.98842245]\n",
      " [ 0.64916801]\n",
      " [ 0.80858427]\n",
      " [ 0.99278939]\n",
      " [ 0.97958416]\n",
      " [ 0.91286802]]\n",
      "21\n",
      "AAPL is Predicted to RISE. +18,-2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 0.79707855], dtype=float32)"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LoadModel()\n",
    "StockHistory = CollectData(\"AAPL\")\n",
    "Predict(\"AAPL\", 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### [F] Prediction for One Stock - Main Block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Predict(stocktopredict, verbosity):\n",
    "    from pandas import read_csv\n",
    "    import pandas as pd\n",
    "    StockToAnalyze = stocktopredict\n",
    "    Analyze(StockToAnalyze, verbosity)\n",
    "\n",
    "    try: \n",
    "        df = read_csv(\"StockData/\"+StockToAnalyze+'.csv')\n",
    "    except IOError:\n",
    "        print(\"Nothing found\")\n",
    "        return 0\n",
    "        \n",
    "    dfrows = df.shape[0]\n",
    "    print(str(dfrows) + \" instances found\")\n",
    "    if (dfrows > 1) and not (df.iloc[0,0] == 'error'):\n",
    "        X_set = df.iloc[0:dfrows-1,[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]].as_matrix() #CHANGED\n",
    "        \n",
    "        model = LoadModel()\n",
    "        Y_pred = model.predict(X_set)\n",
    "        print(Y_pred)\n",
    "\n",
    "        Sum = 0\n",
    "        PosArticles = 0\n",
    "        NegArticles = 0\n",
    "        for i in range(0,Y_pred.shape[0]-1):\n",
    "            if (Y_pred[i] >= 0.5):\n",
    "                PosArticles += 1\n",
    "            else:\n",
    "                NegArticles += 1\n",
    "            Sum += Y_pred[i]\n",
    "        print(len(Y_pred))\n",
    "        finalstockresult = Sum/len(Y_pred)\n",
    "                \n",
    "        if (PosArticles > NegArticles):\n",
    "            print(StockToAnalyze + \" is Predicted to RISE. \" + \"+\" + str(PosArticles) + \",-\" + str(NegArticles))\n",
    "        elif (PosArticles < NegArticles):\n",
    "            print(StockToAnalyze + \" is Predicted to FALL. \" + \"+\" + str(PosArticles) + \",-\" + str(NegArticles))\n",
    "        elif (PosArticles == NegArticles):\n",
    "            print(StockToAnalyze + \" is Predicted to STAY. \" + \"+\" + str(PosArticles) + \",-\" + str(NegArticles))\n",
    "        \n",
    "        return finalstockresult\n",
    "    else:\n",
    "        print(\"Nothing found\")\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### [F] CSV Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Analyze (stockname, verbosity):\n",
    "    import csv\n",
    "    from datetime import datetime, timedelta\n",
    "    TickerName = stockname\n",
    "    URLList = GetNewsURLs(stockname)\n",
    "    StockHistory = CollectData(stockname)\n",
    "    \n",
    "\n",
    "    if (len(URLList) > 0):\n",
    "        #Clear the CSV if it currently exists:\n",
    "        f = open(\"StockData/\"+stockname+'.csv', \"w+\")\n",
    "        f.close()\n",
    "        \n",
    "        #Write in columns headers:\n",
    "        with open(\"StockData/\"+stockname+'.csv', 'a') as csvfile:\n",
    "                filewriter = csv.writer(csvfile, delimiter=',',\n",
    "                                quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "                filewriter.writerow(['Pos','Neg','DaysOld','Mag1','Mag2','Mag3','Mag4','Mag5','Mag6','Mag7','Reg1','Reg2','Reg3','Reg4','dayoftheweek'])\n",
    "        \n",
    "        for i in range(0,len(URLList)-1):\n",
    "            url = URLList[i]\n",
    "            if (verbosity == 1):\n",
    "                print(\"Current URL: \" + url)\n",
    "            articletext, articledate = URLtoText(url)\n",
    "    \n",
    "            #Get Sentiment Value\n",
    "            ListedText = articletext.split() #Turn into list\n",
    "            pos, neg = getSentiment(ListedText)\n",
    "            \n",
    "            if (articledate is None) or (isinstance(articledate, unicode)):\n",
    "                continue\n",
    "            \n",
    "            # Days since Written\n",
    "            today = datetime.now().replace(tzinfo=None)\n",
    "            articledate = articledate.replace(tzinfo=None)\n",
    "            strtoday = today.strftime('%Y-%m-%d')\n",
    "            while BusinessDay(strtoday, stockname) == False:\n",
    "                today = today - timedelta(days=1)\n",
    "                strtoday = today.strftime('%Y-%m-%d')\n",
    "            daysOld = (today - articledate).days\n",
    "    \n",
    "            #Price \"today\"\n",
    "            a = SearchCSVForDate(strtoday, stockname)\n",
    "            pricetoday = StockHistory.get_value(a,\"adjusted_close\")\n",
    "            priceonedayago = StockHistory.get_value(a+1,\"adjusted_close\")\n",
    "            pricetwodaysago = StockHistory.get_value(a+2,\"adjusted_close\")\n",
    "            pricethreedaysago = StockHistory.get_value(a+3,\"adjusted_close\")\n",
    "            pricefourdaysago = StockHistory.get_value(a+4,\"adjusted_close\")\n",
    "            pricefivedaysago = StockHistory.get_value(a+5,\"adjusted_close\")\n",
    "            pricesixdaysago = StockHistory.get_value(a+6,\"adjusted_close\")\n",
    "            pricesevendaysago = StockHistory.get_value(a+7,\"adjusted_close\")\n",
    "    \n",
    "            #Magnitude\n",
    "            mag1 = (pricetoday-priceonedayago)/priceonedayago\n",
    "            mag2 = (pricetoday-pricetwodaysago)/pricetwodaysago\n",
    "            mag3 = (pricetoday-pricethreedaysago)/pricethreedaysago\n",
    "            mag4 = (pricetoday-pricefourdaysago)/pricefourdaysago\n",
    "            mag5 = (pricetoday-pricefivedaysago)/pricefivedaysago\n",
    "            mag6 = (pricetoday-pricesixdaysago)/pricesixdaysago\n",
    "            mag7 = (pricetoday-pricesevendaysago)/pricesevendaysago\n",
    "    \n",
    "            #Regression\n",
    "            reg1 = getSlope(today, 5, 0, stockname) #Past\n",
    "            reg2 = getSlope(today, 10, 0, stockname) #Past\n",
    "            reg3 = getSlope(today, 15, 0, stockname) #Past\n",
    "            reg4 = getSlope(today, 20, 0, stockname) #Past\n",
    "    \n",
    "            #DayOfTheWeek\n",
    "            dayoftheweek = today.weekday() #0=Monday, 6=Sunday - should only be 0-4\n",
    "    \n",
    "            #Log to CSV:\n",
    "            with open(\"StockData/\"+stockname+'.csv', 'a') as csvfile:\n",
    "                filewriter = csv.writer(csvfile, delimiter=',',\n",
    "                                quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "                filewriter.writerow([pos, neg, daysOld, mag1, mag2, mag3, mag4, mag5, mag6, mag7,\n",
    "                             reg1, reg2, reg3, reg4, dayoftheweek])\n",
    "            #END FOR\n",
    "        \n",
    "        \n",
    "    else: #Gives CSV with Error Message\n",
    "        with open(\"StockData/\"+stockname+'.csv', 'a') as csvfile:\n",
    "                filewriter = csv.writer(csvfile, delimiter=',',\n",
    "                                quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "                filewriter.writerow(['error'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getCurrentPrice (stockticker):\n",
    "    import pandas as pd\n",
    "    url = \"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol=\" + stockticker + \"&interval=1min&apikey=\" + \"VPRQOQ85W9KS1DXH\"+\"&datatype=csv\" #apikey \n",
    "    StockData = pd.read_csv(url)\n",
    "    return StockData.iloc[0,4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def LoadModel():\n",
    "    from keras.models import model_from_json\n",
    "    json_file = open('SavedModel/new2model.json', 'r')\n",
    "    loaded_model_json = json_file.read()\n",
    "    json_file.close()\n",
    "    loaded_model = model_from_json(loaded_model_json)\n",
    "\n",
    "    loaded_model.load_weights(\"SavedModel/new2model.h5\")\n",
    "    print(\"Loaded model from disk\")\n",
    "    return loaded_model"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}