Google2Csv.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Google2Csv.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyN/EIlGRKp/NPVnoqRZtLV/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/psalias2006/Google2Csv/blob/master/Google2Csv.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3w-fyktSoKxu",
        "colab_type": "text"
      },
      "source": [
        "##GoogleScraper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zCQtYO-Po_kf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2d2221b1-b425-4723-adbc-7aa415ecf254"
      },
      "source": [
        "!pip install beautifulsoup4\n",
        "\n",
        "from bs4 import BeautifulSoup\n",
        "import requests\n",
        "import pandas as pd"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (4.6.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6JZjJa-joXEh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def simpleGoogleSearch(query, start):\n",
        "  results = []\n",
        "\n",
        "  query = query.replace(' ', '+')\n",
        "  URL = f\"https://google.com/search?q={query}&start={start}\"\n",
        "\n",
        "  # desktop user-agent\n",
        "  USER_AGENT = \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0\"\n",
        "  \n",
        "  headers = {\"user-agent\" : USER_AGENT}\n",
        "  resp = requests.get(URL, headers=headers)\n",
        "\n",
        "  if resp.status_code == 200:\n",
        "    soup = BeautifulSoup(resp.content, \"html.parser\")\n",
        "\n",
        "    for g in soup.find_all('div', class_='r'):\n",
        "      anchors = g.find_all('a')\n",
        "\n",
        "      if anchors:\n",
        "        link  = anchors[0]['href']\n",
        "        title = g.find('h3').text\n",
        "        item  = {\"title\": title, \"link\": link}\n",
        "        results.append(item)\n",
        "\n",
        "  return results"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kBCwCEcGob2M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def googleToPandas(googleQuery):\n",
        "  resultsCounter  = 0\n",
        "  resultsList     = []\n",
        "\n",
        "  while True:\n",
        "    pageResults = simpleGoogleSearch(googleQuery, resultsCounter)\n",
        "    \n",
        "    if not pageResults: break\n",
        "    else: \n",
        "      resultsList.extend(pageResults)\n",
        "      resultsCounter = resultsCounter + 10\n",
        "\n",
        "  return pd.DataFrame(resultsList)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SoUutC4DojV7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "googleSearchQuery   = \"Covid 19\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r44AZDaBpjGO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "d7a82dce-8463-45fa-c088-8cac5dba3fa5"
      },
      "source": [
        "results = googleToPandas(googleSearchQuery)\n",
        "results"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>link</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Coronavirus Disease 2019 (COVID-19) | CDC</td>\n",
              "      <td>https://www.cdc.gov/coronavirus/2019-ncov/inde...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Cases in the U.S. | CDC</td>\n",
              "      <td>https://www.cdc.gov/coronavirus/2019-ncov/case...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Coronavirus (COVID-19) frequently asked questi...</td>\n",
              "      <td>https://www.cdc.gov/coronavirus/2019-ncov/faq....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Coronavirus disease 2019 - World Health Organi...</td>\n",
              "      <td>https://www.who.int/emergencies/diseases/novel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Coronavirus (COVID-19) events as they happen</td>\n",
              "      <td>https://www.who.int/emergencies/diseases/novel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231</th>\n",
              "      <td>This web page is parked FREE, courtesy of GoDa...</td>\n",
              "      <td>http://covid-2019.me/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232</th>\n",
              "      <td>COVID-19 - Doctors eBook</td>\n",
              "      <td>http://doctorsebook.com/covid-19/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>233</th>\n",
              "      <td>COVID-19 Update</td>\n",
              "      <td>https://www.oepreview.com/covid19sample/covid-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>234</th>\n",
              "      <td>Page 2 – Corona Virus (Covid-19) - Covid-19 Ou...</td>\n",
              "      <td>https://covid-19outbreak.co/page/2/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>235</th>\n",
              "      <td>This web page is parked FREE, courtesy of GoDa...</td>\n",
              "      <td>http://covid-2019.me/</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>236 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 title                                               link\n",
              "0            Coronavirus Disease 2019 (COVID-19) | CDC  https://www.cdc.gov/coronavirus/2019-ncov/inde...\n",
              "1                              Cases in the U.S. | CDC  https://www.cdc.gov/coronavirus/2019-ncov/case...\n",
              "2    Coronavirus (COVID-19) frequently asked questi...  https://www.cdc.gov/coronavirus/2019-ncov/faq....\n",
              "3    Coronavirus disease 2019 - World Health Organi...  https://www.who.int/emergencies/diseases/novel...\n",
              "4         Coronavirus (COVID-19) events as they happen  https://www.who.int/emergencies/diseases/novel...\n",
              "..                                                 ...                                                ...\n",
              "231  This web page is parked FREE, courtesy of GoDa...                              http://covid-2019.me/\n",
              "232                           COVID-19 - Doctors eBook                  http://doctorsebook.com/covid-19/\n",
              "233                                    COVID-19 Update  https://www.oepreview.com/covid19sample/covid-...\n",
              "234  Page 2 – Corona Virus (Covid-19) - Covid-19 Ou...                https://covid-19outbreak.co/page/2/\n",
              "235  This web page is parked FREE, courtesy of GoDa...                              http://covid-2019.me/\n",
              "\n",
              "[236 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y-wbjJuYqnOT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results.to_csv('GoogleResults.csv', index=False)\n",
        "results.to_excel('GoogleResults.xlsx', index=False)\n",
        "results.to_json('GoogleResults.jsonl', orient='records', lines=True)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}