Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bc14e4f-c4ac-46c1-a74e-220d6ff8a667",
   "metadata": {},
   "outputs": [],
   "source": [
    "from flask import Flask\n",
    "from flask_restful import Resource, Api, reqparse\n",
    "import pandas as pd\n",
    "\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "from time import sleep\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "\n",
    "app = Flask(__name__)\n",
    "api = Api(app)\n",
    "\n",
    "class Results(Resource):\n",
    "    def get(self):\n",
    "        # data = scrape_results()  # scrape results\n",
    "        return {'test': 'zap'}, 200\n",
    "        # return {'data': data.to_json()}, 200  # return data as JSON object and 200 OK\n",
    "    \n",
    "api.add_resource(Results, '/results')  # add endpoint\n",
    "\n",
    "TOKENS = {'89b638c0524d4d688b56ff77add5f79c':'Tom',\n",
    "          'bc2862f420844334a08e36b47119df75':'Sammy',\n",
    "          'baef0dbf8c8f4d978ef962ea9a0277f3':'Chris'}\n",
    "\n",
    "def scrape_results():\n",
    "    master=[]\n",
    "    for z in TOKENS.keys():\n",
    "        s=Service(ChromeDriverManager().install())\n",
    "        driver = webdriver.Chrome(service=s)\n",
    "        driver.implicitly_wait(100)\n",
    "        driver.get(\"https://www.17lands.com/user_history/{}\".format(z))\n",
    "        sleep(10)\n",
    "        html = driver.execute_script(\"return document.getElementsByTagName('tbody')[0].innerHTML\")\n",
    "        soup = BeautifulSoup(html, \"html.parser\")\n",
    "        for x in range(len(soup.find_all('tr'))):\n",
    "            tr= soup.find_all('tr')[x] #tr is the tag the deliniates distinct drafts\n",
    "            td=tr.find_all('td') #td gives us the individual parts. because td has type Resultset, we can't find_all('a') and have to use regex\n",
    "            data = [x.string for x in td] #if one of the elements has multiple tags or no obvious string, it returns None\n",
    "            if 'PremierDraft' not in data:\n",
    "                continue\n",
    "            if len(data) != 9:\n",
    "                continue\n",
    "            data=list(filter(None, data)) #remove Nones from the list\n",
    "            #data.pop(3) #removes Format\n",
    "            data.insert(3,data[2][-1]) #adds losses after record\n",
    "            data[2]=data[2][0] #converts record into wins\n",
    "            if len(data) != 7:\n",
    "                continue\n",
    "            try:\n",
    "                data.append(re.search(pattern='title=\"(.*?)\"', string=str(td[3])).group(1)) #finds colors and appends it to the list\n",
    "            except:\n",
    "                continue\n",
    "            links = re.findall(pattern='<a href=\"(.*?)</a>', string=str(td[-1])) #finds each link string\n",
    "            links = ['https://www.17lands.com/'+links[x] for x in range(len(links))] #completes the link\n",
    "            links = [links[x].split('>') for x in range(len(links))] #splits the link from the description\n",
    "            for x in range(len(links)):\n",
    "                           links[x]= [links[x][1], links[x][0]] #inverts the order of the description and link to prep for being a dictionary\n",
    "            links_dict = dict(links) #turns list of lists into dictionary with descriptions as keys and links as values\n",
    "            data.append(links_dict) #adds the dictionary to the list\n",
    "            data.append(TOKENS[z])\n",
    "            master.append(data)\n",
    "        driver.close()\n",
    "    df = pd.DataFrame(master, columns = ['Date','Set','Wins','Losses','Format','Start Rank', 'End Rank', 'Colors', 'Links', 'Pilot'])\n",
    "    df['Draft'] = [df['Links'][x]['Draft'] if 'Draft' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df['Pool'] = [df['Links'][x]['Pool'] if 'Pool' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df['Details'] = [df['Links'][x]['Details'] if 'Details' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df['Deck 1'] = [df['Links'][x]['Deck 1'] if 'Deck 1' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df['Deck 2'] = [df['Links'][x]['Deck 2'] if 'Deck 2' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df['Deck 3'] = [df['Links'][x]['Deck 3'] if 'Deck 3' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df['Deck 4'] = [df['Links'][x]['Deck 4'] if 'Deck 4' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df['Deck 5'] = [df['Links'][x]['Deck 5'] if 'Deck 5' in df['Links'][x].keys() else None for x in range(len(df))]\n",
    "    df.drop(columns=['Links'], inplace=True)\n",
    "    return df\n",
    "\n",
    "#if __name__ == '__main__':\n",
    "#    app.run()  # run our Flask app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03fe73d9-8761-47a4-8afb-3f98a606d805",
   "metadata": {},
   "outputs": [],
   "source": [
    "scrape_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1cf6b3d-e583-469c-b0cc-aeb1294f5b2d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}