crawler.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Email Crawler\n",
    "Crawler script designed to download directories of hospital websites to get staff email addresses. To add a new hospital to crawl extend the visitor class to implement the functions `filter_url` and `crawl_name_title_email_prorgam` to start crawling a new website.\n",
    "\n",
    "There is some trail and error involved with inspecting the DOM on the profile page to implement `crawl_name_title_email_prorgam`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bs4 in /usr/local/lib/python3.7/site-packages (from -r requirements.txt (line 1)) (0.0.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/site-packages (from -r requirements.txt (line 2)) (1.18.4)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.7/site-packages (from -r requirements.txt (line 3)) (1.0.4)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (2.22.0)\n",
      "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/site-packages (from bs4->-r requirements.txt (line 1)) (4.8.2)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/site-packages (from pandas->-r requirements.txt (line 3)) (2.8.0)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/site-packages (from pandas->-r requirements.txt (line 3)) (2020.1)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.7/site-packages (from requests->-r requirements.txt (line 4)) (2.8)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/site-packages (from requests->-r requirements.txt (line 4)) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests->-r requirements.txt (line 4)) (2019.9.11)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/site-packages (from requests->-r requirements.txt (line 4)) (1.25.6)\n",
      "Requirement already satisfied: soupsieve>=1.2 in /usr/local/lib/python3.7/site-packages (from beautifulsoup4->bs4->-r requirements.txt (line 1)) (2.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas->-r requirements.txt (line 3)) (1.12.0)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import html\n",
    "import re\n",
    "import sys\n",
    "import urllib.request as urllib2\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import requests.exceptions\n",
    "from urllib.parse import urlsplit\n",
    "from collections import deque\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import shutil\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "class InstiSiteVisitor(object):\n",
    "    def __init__(self, root, name, shortname):\n",
    "        '''\n",
    "        @root: the root URL to crawl.\n",
    "        @name: institution name.\n",
    "        '''\n",
    "        self.root = root\n",
    "        self.name = name\n",
    "        self.shortname = shortname\n",
    "\n",
    "    def print_info(self):\n",
    "        print('Name : ' + self.name + \n",
    "              ', root url : ' + self.root + \n",
    "              ', short name : ' + self.shortname)\n",
    "\n",
    "    def filter_url(self, link):\n",
    "        return 0\n",
    "    \n",
    "    def crawl_name_title_email_program(self, soup):\n",
    "        '''\n",
    "        @soup : beautifulsoup object that represents \n",
    "                the profile page of the doctor in the directory\n",
    "        '''\n",
    "        return ['', '', '', '']\n",
    "    \n",
    "    def name(self):\n",
    "        return self.name\n",
    "    \n",
    "    def root(self):\n",
    "        return self.root\n",
    "    \n",
    "    def get_mailto_ref(self, soup):\n",
    "        email = \"\"\n",
    "        a = None\n",
    "        for anchor in soup.find_all(\"a\"):\n",
    "            # find emails\n",
    "            link = anchor.attrs[\"href\"] if \"href\" in anchor.attrs else ''\n",
    "            if link.find(\"ailto\") >= 0:\n",
    "                a = anchor\n",
    "                email = link.split(':')[1]\n",
    "                yield [email, a]\n",
    "\n",
    "class BronxCareVisitor(InstiSiteVisitor):\n",
    "    def __init__(self):\n",
    "        super().__init__('https://www.bronxcare.org/our-services/pediatrics/', \n",
    "                         'BronxCare Health System', \n",
    "                         'bronxcare')\n",
    "\n",
    "    def filter_url(self, link):\n",
    "        if link.find('www.bronxcare.org/physicians/find-a-physician/detail') >= 0:\n",
    "            return 1\n",
    "        if link == 'https://www.bronxcare.org/physicians/find-a-physician/detail/':\n",
    "            return 0\n",
    "        super().filter_url(link)\n",
    "        \n",
    "    def crawl_name_title_email_program(self, soup):\n",
    "        div = soup.find('div', {'class':'description'})\n",
    "        name = ''\n",
    "        email = ''\n",
    "        title = ''\n",
    "        program = ''\n",
    "        \n",
    "        if div:\n",
    "            h3 = div.find('h3')\n",
    "            name_and_title = h3.text.split(\",\", 1)\n",
    "            name = name_and_title[0].strip()\n",
    "            title = name_and_title[1].strip()\n",
    "            \n",
    "        div = soup.find('div', {'class': 'physician-app profile'})\n",
    "        if div:\n",
    "            for [e, a] in super().get_mailto_ref(div):\n",
    "                email = a.text\n",
    "                break\n",
    "        return [name, title, email, program]\n",
    "\n",
    "class CincinattiVisitor(InstiSiteVisitor):\n",
    "    def __init__(self):\n",
    "        super().__init__('https://www.cincinnatichildrens.org/search/doctor-search?q=&start=0', \n",
    "                         'Cincinnati Children\\'s Hospital',\n",
    "                         'cincinnati')\n",
    "\n",
    "    def crawl_name_title_email_program(self, soup):\n",
    "        name = \"\"\n",
    "        title = \"\"\n",
    "        email = \"\"\n",
    "        program = \"\"\n",
    "        h1 = soup.find(\"h1\", {\"class\":\"person-name\"})\n",
    "        if h1:\n",
    "            parts = h1.text.split(\",\", 1)\n",
    "            name = parts[0]\n",
    "            if len(parts) >= 2:\n",
    "                title = parts[1]\n",
    "            for [e, a] in self.get_mailto_ref(soup):\n",
    "                email = e\n",
    "                break\n",
    "        return [name, title, email, program]\n",
    "    \n",
    "    def filter_url(self, link):\n",
    "        if link.find('https://www.cincinnatichildrens.org/bio/') >= 0:\n",
    "            return 1\n",
    "        if link.find('https://www.cincinnatichildrens.org/search/doctor-search') >= 0:\n",
    "            return 1\n",
    "        return super().filter_url(link)\n",
    "\n",
    "class UFL_HSCJ_Visitor(InstiSiteVisitor):\n",
    "    def __init__(self):\n",
    "        super().__init__('https://hscj.ufl.edu/pediatrics/faculty.aspx', \n",
    "                         'University of Florida',\n",
    "                         'ufl_hscj')\n",
    "    \n",
    "    def filter_url(self, link):\n",
    "        if link.find('https://hscj.ufl.edu/directory/bio/') >=0 :\n",
    "            return 1\n",
    "        return super().filter_url(link)\n",
    "    \n",
    "    def crawl_name_title_email_program(self, soup):\n",
    "        name = \"\"\n",
    "        title = \"\"\n",
    "        email = \"\"\n",
    "        program = \"\"\n",
    "        title_div = soup.find(\"div\", {\"class\":\"titleDiv\"})\n",
    "        if title_div:\n",
    "            splits = title_div.h1.text.split(\",\", 1)\n",
    "            name = splits[0]\n",
    "            title = splits[1]\n",
    "\n",
    "        for anchor in soup.find_all(\"a\"):\n",
    "            # find emails\n",
    "            link = anchor.attrs[\"href\"] if \"href\" in anchor.attrs else ''\n",
    "            if link.find(\"mailto\") >= 0:\n",
    "                parent_div = anchor.find_parent(\"div\")\n",
    "                if \"infoLeft\" in parent_div.attrs[\"class\"] :\n",
    "                    email = link.split(\":\")[1]\n",
    "        return [name, title, email, program]\n",
    "\n",
    "class BostonChildrenVisitor(InstiSiteVisitor):\n",
    "    def __init__(self):\n",
    "        super().__init__('http://www.childrenshospital.org/directory#sort='\\\n",
    "                         'relevancy&f:_3BD935D7-4C32-43AD-B7CA-5F366D2450F8=[Doctor]', \n",
    "                         'Boston Children\\'s Hospital',\n",
    "                         'boston_children')\n",
    "\n",
    "    def filter_url(self, link):\n",
    "        if link.find('http://www.childrenshospital.org/directory/physicians/') >= 0:\n",
    "            return 1\n",
    "        if link.find('http://www.childrenshospital.org/directory#') >= 0:\n",
    "            return 1\n",
    "        return super().filter_url(link)\n",
    "    \n",
    "    def crawl_name_title_email_boston(self, soup):\n",
    "        name = \"\"\n",
    "        title = \"\"\n",
    "        email = \"\"\n",
    "        program = \"\"\n",
    "        div = soup.find(\"div\", {\"class\":\"doctor-info col-xs-12 col-sm-8\"})\n",
    "        if div:\n",
    "            print(div)\n",
    "            \n",
    "        return [name, title, email, program]\n",
    "\n",
    "class Crawler(object):\n",
    "    def __init__(self, df, insti_visitor, max_pages_to_crawl, crawler_timeout_s):\n",
    "        '''\n",
    "        @urls: a string containing the (comma separated) URLs to crawl\n",
    "        @insti_visitor: visitor object related to the institution\n",
    "        @max_pages_to_crawl: max pages after which the crawler should stop crawling\n",
    "        @crawler_timeout_s: timeout for the get requests\n",
    "        '''\n",
    "        self.df = df\n",
    "        self.insti_visitor = insti_visitor\n",
    "        self.urls = self.insti_visitor.root.split(',')\n",
    "        self.max_pages_to_crawl = max_pages_to_crawl\n",
    "        self.timeout = crawler_timeout_s\n",
    "\n",
    "    def crawl(self):\n",
    "        '''\n",
    "        Iterate the list of URLs and request each page, then parse it and \n",
    "        print the emails we find. \n",
    "        '''\n",
    "        # a queue of urls to be crawled\n",
    "        new_urls = deque(self.urls)\n",
    "        processed_urls = set()\n",
    "        emails = set()\n",
    "        n = max_pages_to_crawl\n",
    "\n",
    "        while new_urls and n > 0:\n",
    "            n = n - 1\n",
    "            url = new_urls.popleft()\n",
    "            print('Processing : ' + url)\n",
    "\n",
    "            parts = urlsplit(url)\n",
    "            base_url = \"{0.scheme}://{0.netloc}\".format(parts)\n",
    "            path = url[:url.rfind('/') + 1] if '/' in parts.path else url\n",
    "            regex = r'mailto:.*.edu\">'\n",
    "\n",
    "            try:\n",
    "                response = requests.get(url) #, params={'timeout':self.timeout})\n",
    "            except (requests.exceptions.MissingSchema, \n",
    "                    requests.exceptions.ConnectionError,\n",
    "                    requests.exceptions.InvalidURL):\n",
    "                print('Skipping : ' + url)\n",
    "                continue\n",
    "            processed_urls.add(url)\n",
    "\n",
    "            soup = BeautifulSoup(response.text, \"lxml\") # soup from page\n",
    "            [name, title, email, program] = \\\n",
    "                self.insti_visitor.crawl_name_title_email_program(soup)\n",
    "            if name:\n",
    "                self.df = self.df.append({'Name': name,\n",
    "                                          'Title': title,\n",
    "                                          'Email': email,\n",
    "                                          'Program': program,\n",
    "                                          'Institution': self.insti_visitor.name}, \n",
    "                                         ignore_index=True)\n",
    "                continue\n",
    "\n",
    "            # find and process all the anchors in the document\n",
    "            for anchor in soup.find_all(\"a\"):\n",
    "                # extract link url from the anchor\n",
    "                link = anchor.attrs[\"href\"] if \"href\" in anchor.attrs else ''\n",
    "                # resolve relative links\n",
    "                if link.startswith('/'):\n",
    "                    link = base_url + link\n",
    "                elif not link.startswith('http'):\n",
    "                    link = path + link\n",
    "                # add the new url to the queue if it was not enqueued nor processed yet\n",
    "                if not link in new_urls and not link in processed_urls:\n",
    "                    if self.insti_visitor.filter_url(link) == 1:\n",
    "                        new_urls.append(link)\n",
    "        return self.df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Config setup for the script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pick the visitor based on the insitution you want to crawl from!\n",
    "insti = CincinattiVisitor()\n",
    "max_pages_to_crawl = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing : https://www.cincinnatichildrens.org/search/doctor-search?q=&start=0\n"
     ]
    },
    {
     "ename": "FeatureNotFound",
     "evalue": "Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFeatureNotFound\u001b[0m                           Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-0f954d3ff96a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_columns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mcrawler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCrawler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minsti\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_pages_to_crawl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrawler_timeout_s\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcrawler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrawl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop_duplicates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0memails_output_folder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhome\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Desktop\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-04c1c3e4ddc0>\u001b[0m in \u001b[0;36mcrawl\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    219\u001b[0m             \u001b[0mprocessed_urls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m             \u001b[0msoup\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"lxml\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# soup from page\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m             \u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0memail\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprogram\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsti_visitor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrawl_name_title_email_program\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msoup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/bs4/__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m                     \u001b[0;34m\"Couldn't find a tree builder with the features you \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    227\u001b[0m                     \u001b[0;34m\"requested: %s. Do you need to install a parser library?\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 228\u001b[0;31m                     % \",\".join(features))\n\u001b[0m\u001b[1;32m    229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m         \u001b[0;31m# At this point either we have a TreeBuilder instance in\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFeatureNotFound\u001b[0m: Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?"
     ]
    }
   ],
   "source": [
    "output_columns=['Name', 'Title', 'Email', 'Program', 'Institution']\n",
    "df = pd.DataFrame(columns=output_columns)\n",
    "crawler = Crawler(df, insti, max_pages_to_crawl, crawler_timeout_s=5)\n",
    "df = crawler.crawl()\n",
    "df.drop_duplicates()\n",
    "emails_output_folder = os.path.join(str(Path.home()), \"Desktop\")\n",
    "emails_output = emails_output_folder + '/emails_' + insti.shortname + '.csv'\n",
    "df.to_csv(emails_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}