Downloading_Kaggle_Datasets_in_Colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Downloading Kaggle Datasets in Colab.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/connected-bsamadi/colab-notebooks/blob/master/Downloading_Kaggle_Datasets_in_Colab.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "iMH6Oz9OrdSU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Training Dataset\n",
        "Before running this notebook, go to \"My Account\" on Kaggle and press \"Create New API Token\", and upload the kaggle.json file to your Google Drive.\n",
        "\n",
        "reference: [This notebook provides a recipe for using Kaggle API with Colab](https://colab.research.google.com/drive/1eufc8aNCdjHbrBhuy7M7X6BGyzAyRbrF)"
      ]
    },
    {
      "metadata": {
        "id": "vhiQlk2T4tWC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Installing Kaggle API"
      ]
    },
    {
      "metadata": {
        "id": "ReOLDD542gkn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install kaggle >/dev/null"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "r-D0YrT_yOwz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from googleapiclient.discovery import build\n",
        "import io, os\n",
        "import zipfile\n",
        "from googleapiclient.http import MediaIoBaseDownload\n",
        "from google.colab import auth, files\n",
        "import pandas as pd\n",
        "\n",
        "auth.authenticate_user()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gP0VjAf10MZp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Find the kaggle.json on Google Drive\n",
        "drive_service = build('drive', 'v3')\n",
        "results = drive_service.files().list(\n",
        "        q=\"name = 'kaggle.json'\", fields=\"files(id)\").execute()\n",
        "kaggle_api_key = results.get('files', [])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uzlWVVxX0SUN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "97387de3-99fc-4266-907c-d62c4e92f052"
      },
      "cell_type": "code",
      "source": [
        "# Write kaggle.json to ~/.kaggle/\n",
        "filename = \"/root/.kaggle/kaggle.json\"\n",
        "os.makedirs(os.path.dirname(filename), exist_ok=True)\n",
        "request = drive_service.files().get_media(fileId=kaggle_api_key[0]['id'])\n",
        "fh = io.FileIO(filename, 'wb')\n",
        "downloader = MediaIoBaseDownload(fh, request)\n",
        "done = False\n",
        "while done is False:\n",
        "    status, done = downloader.next_chunk()\n",
        "    print(\"Download %d%%.\" % int(status.progress() * 100))\n",
        "os.chmod(filename, 600)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Download 100%.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ym86XRoH1qg0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "outputId": "6f7a6187-934b-4acb-8a27-3a3ad66c4cbb"
      },
      "cell_type": "code",
      "source": [
        "!kaggle datasets list"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ref                                                      title                                               size  lastUpdated          downloadCount  \n",
            "-------------------------------------------------------  -------------------------------------------------  -----  -------------------  -------------  \n",
            "center-for-policing-equity/data-science-for-good         Data Science for Good: Center for Policing Equity    6MB  2018-10-02 17:38:13           1048  \n",
            "lava18/google-play-store-apps                            Google Play Store Apps                               2MB  2018-09-18 20:49:49           6467  \n",
            "blastchar/telco-customer-churn                           Telco Customer Churn                               172KB  2018-02-23 18:20:00           7557  \n",
            "jenny18/honey-bee-annotated-images                       Annotated Honey Bee Images                          50MB  2018-09-16 23:42:12            588  \n",
            "lucidlenn/sloan-digital-sky-survey                       Sloan Digital Sky Survey RD14                      446KB  2018-09-20 00:23:41            893  \n",
            "neuromusic/avocado-prices                                Avocado Prices                                     629KB  2018-06-06 05:28:35           9629  \n",
            "san-francisco/sf-police-calls-for-service-and-incidents  SF Police Calls for Service and Incidents          165MB  2018-10-04 09:36:58            948  \n",
            "fmena14/volcanoesvenus                                   Volcanoes on Venus                                 109MB  2018-08-25 01:37:34            346  \n",
            "census/census-bureau-usa                                 United States Census                               161MB  2018-04-17 18:52:46              0  \n",
            "bigquery/patents                                         Google Patents Public Data                           2TB  2018-09-19 01:57:26              0  \n",
            "cityofLA/los-angeles-metro-bike-share-trip-data          Los Angeles Metro Bike Share Trip Data               3MB  2018-09-06 06:37:25           1316  \n",
            "kevinarvai/clinvar-conflicting                           Genetic Variant Classifications                      3MB  2018-04-22 19:16:07           1673  \n",
            "chicago/chicago-red-light-and-speed-camera-data          Chicago Red Light and Speed Camera Data             22MB  2018-10-02 16:41:44            493  \n",
            "smid80/weatherww2                                        Weather Conditions in World War Two                  2MB  2017-11-01 05:05:30           3194  \n",
            "cityofLA/la-restaurant-market-health-data                LA Restaurant & Market Health Data                  10MB  2018-09-06 05:37:59           2357  \n",
            "kmader/nih-deeplesion-subset                             NIH DeepLesion Subset                                8GB  2018-07-22 18:58:32            263  \n",
            "census/total-business-inventories-and-sales-data         Total Business Inventories and Sales Data           20KB  2018-10-04 09:33:46            516  \n",
            "marklvl/bike-sharing-dataset                             Bike Sharing in Washington D.C. Dataset            273KB  2018-05-28 11:31:09           2022  \n",
            "chicago/chicago-311-service-requests                     Chicago 311 Service Requests                       232MB  2018-10-02 16:08:44            314  \n",
            "census/total-construction-spending-data-collection       Total Construction Spending Data Collection         33KB  2018-10-04 09:34:56            286  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wjP5cOwj4x6W",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Downloading the Dataset"
      ]
    },
    {
      "metadata": {
        "id": "AEc8uvrQS60Z",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset = 'jigsaw-toxic-comment-classification-challenge'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pZZcXT1_24nv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "folder = '/content/'+dataset\n",
        "if not os.path.isdir(folder):\n",
        "  os.mkdir(folder)\n",
        "os.chdir(folder)\n",
        "os.system('kaggle competitions download -c '+dataset)\n",
        "for file in os.listdir():\n",
        "  filename, file_extension = os.path.splitext(file)\n",
        "  if file_extension == '.zip':\n",
        "    zip_ref = zipfile.ZipFile(file, 'r')\n",
        "    zip_ref.extractall()\n",
        "    zip_ref.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eyBEsGJCTIR_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "69d09fe8-c901-47d6-d751-1d6479ff185a"
      },
      "cell_type": "code",
      "source": [
        "train_df=pd.read_csv('train.csv')\n",
        "train_df.head()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0001b41b1c6bb37e</td>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0001d958c54c6e35</td>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id                                       comment_text  toxic  \\\n",
              "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
              "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
              "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
              "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
              "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
              "\n",
              "   severe_toxic  obscene  threat  insult  identity_hate  \n",
              "0             0        0       0       0              0  \n",
              "1             0        0       0       0              0  \n",
              "2             0        0       0       0              0  \n",
              "3             0        0       0       0              0  \n",
              "4             0        0       0       0              0  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "metadata": {
        "id": "-TnlPo4bx-qT",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Converting the dataset to the Google Cloud AutoML CSV format"
      ]
    },
    {
      "metadata": {
        "id": "BPCL0wQulHNy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Remove newline characters\n",
        "def remove_newline(txt):\n",
        "  txt_maxlen = 10000-2\n",
        "  if len(txt) > txt_maxlen:\n",
        "    qtxt = txt[0:txt_maxlen]\n",
        "  else:\n",
        "    qtxt = txt\n",
        "  qtxt = '\\\"'+qtxt.replace('\\\"',' ')+'\\\"'\n",
        "  qtxt = qtxt.replace('\\r',' ')\n",
        "  qtxt = qtxt.replace('\\n',' ')\n",
        "  qtxt = qtxt.replace('\\t',' ')\n",
        "  qtxt = qtxt.replace('#',' ')\n",
        "  return qtxt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5UJ3EbA3xTal",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "88868318-b3ca-4bf3-dc57-1f306d48f318"
      },
      "cell_type": "code",
      "source": [
        "labels=[i for i in train_df][2:]\n",
        "labels"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "metadata": {
        "id": "YtHT7zy-T3Cu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "automl_filename = dataset.replace('-','_')+'.csv'\n",
        "automl_file = open(automl_filename,'wt')\n",
        "for index, item in train_df.iterrows():\n",
        "  line = remove_newline(item['comment_text'])\n",
        "  safe=1\n",
        "  for tag in labels:\n",
        "    if item[tag]:\n",
        "      line=line+','+tag\n",
        "      safe=0\n",
        "  if safe:\n",
        "    line=line+',safe'\n",
        "  line=line+'\\n'\n",
        "  automl_file.write(line)\n",
        "automl_file.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1akYs3PpUall",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "files.download(automl_filename)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rH8Otnxt0ZsW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0aa6a770-6bb0-4710-a2c8-7222e71547ad"
      },
      "cell_type": "code",
      "source": [
        "os.system('rm '+automl_filename)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    }
  ]
}