4.Spark_GCS_connection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3307b886",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.conf import SparkConf\n",
    "from pyspark.context import SparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9f0ddbff",
   "metadata": {},
   "outputs": [],
   "source": [
    "credentials_location = '/home/tez/.google/credentials/google_credentials.json'\n",
    "\n",
    "conf = SparkConf() \\\n",
    "    .setMaster('local[*]') \\\n",
    "    .setAppName('test') \\\n",
    "    .set(\"spark.jars\", \"./lib/gcs-connector-hadoop3-2.2.5.jar\") \\\n",
    "    .set(\"spark.hadoop.google.cloud.auth.service.account.enable\", \"true\") \\\n",
    "    .set(\"spark.hadoop.google.cloud.auth.service.account.json.keyfile\", credentials_location)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b83404e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: An illegal reflective access operation has occurred\n",
      "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/home/tez/spark/spark-3.1.3-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.3.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "22/10/11 00:44:09 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "22/10/11 00:44:11 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n"
     ]
    }
   ],
   "source": [
    "sc = SparkContext(conf=conf)\n",
    "\n",
    "hadoop_conf = sc._jsc.hadoopConfiguration()\n",
    "\n",
    "hadoop_conf.set(\"fs.AbstractFileSystem.gs.impl\",  \"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS\")\n",
    "hadoop_conf.set(\"fs.gs.impl\", \"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem\")\n",
    "hadoop_conf.set(\"fs.gs.auth.service.account.json.keyfile\", credentials_location)\n",
    "hadoop_conf.set(\"fs.gs.auth.service.account.enable\", \"true\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c4713e2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder \\\n",
    "    .config(conf=sc.getConf()) \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1ee1eb1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_green = spark.read.parquet('gs://tez-bucket-spark/pq/green/*/*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "104b40ab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2802931"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_green.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f56a885d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 3:>                                                          (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+---------------------+------------------+----------+------------+------------+---------------+-------------+-----------+-----+-------+----------+------------+---------+---------------------+------------+------------+---------+--------------------+\n",
      "|VendorID|lpep_pickup_datetime|lpep_dropoff_datetime|store_and_fwd_flag|RatecodeID|PULocationID|DOLocationID|passenger_count|trip_distance|fare_amount|extra|mta_tax|tip_amount|tolls_amount|ehail_fee|improvement_surcharge|total_amount|payment_type|trip_type|congestion_surcharge|\n",
      "+--------+--------------------+---------------------+------------------+----------+------------+------------+---------------+-------------+-----------+-----+-------+----------+------------+---------+---------------------+------------+------------+---------+--------------------+\n",
      "|       2| 2019-12-18 15:52:30|  2019-12-18 15:54:39|                 N|       1.0|         264|         264|            5.0|          0.0|        3.5|  0.5|    0.5|      0.01|         0.0|     null|                  0.3|        4.81|         1.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:45:58|  2020-01-01 00:56:39|                 N|       5.0|          66|          65|            2.0|         1.28|       20.0|  0.0|    0.0|      4.06|         0.0|     null|                  0.3|       24.36|         1.0|      2.0|                 0.0|\n",
      "|       2| 2020-01-01 00:41:38|  2020-01-01 00:52:49|                 N|       1.0|         181|         228|            1.0|         2.47|       10.5|  0.5|    0.5|      3.54|         0.0|     null|                  0.3|       15.34|         1.0|      1.0|                 0.0|\n",
      "|       1| 2020-01-01 00:52:46|  2020-01-01 01:14:21|                 N|       1.0|         129|         263|            2.0|          6.3|       21.0| 3.25|    0.5|       0.0|         0.0|     null|                  0.3|       25.05|         2.0|      1.0|                2.75|\n",
      "|       1| 2020-01-01 00:19:57|  2020-01-01 00:30:56|                 N|       1.0|         210|         150|            1.0|          2.3|       10.0|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        11.3|         1.0|      1.0|                 0.0|\n",
      "|       1| 2020-01-01 00:52:33|  2020-01-01 01:09:54|                 N|       1.0|          35|          39|            1.0|          3.0|       13.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        14.8|         1.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:10:18|  2020-01-01 00:22:16|                 N|       1.0|          25|          61|            1.0|         2.77|       11.0|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        12.3|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 01:03:14|  2020-01-01 01:29:45|                 N|       1.0|         225|          89|            1.0|         4.98|       20.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        21.8|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:04:11|  2020-01-01 00:09:48|                 N|       1.0|         129|         129|            1.0|         0.71|        5.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|         6.8|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:25:52|  2020-01-01 00:32:16|                 N|       1.0|         129|          83|            1.0|          0.8|        5.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|         6.8|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:47:32|  2020-01-01 00:59:25|                 N|       1.0|          82|         173|            1.0|         1.52|        9.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        10.8|         2.0|      1.0|                 0.0|\n",
      "|       1| 2020-01-01 00:26:40|  2020-01-01 00:40:42|                 N|       1.0|          74|          69|            1.0|          3.8|       14.0|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        15.3|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:38:47|  2020-01-01 00:46:02|                 N|       1.0|          74|          41|            1.0|         1.12|        6.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|         7.8|         1.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:52:18|  2020-01-01 01:09:58|                 N|       1.0|          41|         127|            1.0|         5.67|       19.0|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        20.3|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:16:01|  2020-01-01 00:26:40|                 N|       1.0|           7|         260|            1.0|         1.86|        9.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        10.8|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:21:15|  2020-01-01 00:28:03|                 N|       1.0|           7|           7|            1.0|         1.42|        7.0|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|         8.3|         2.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:35:42|  2020-01-01 01:02:00|                 N|       1.0|           7|         133|            1.0|        15.48|       43.0|  0.5|    0.5|      8.86|         0.0|     null|                  0.3|       53.16|         1.0|      1.0|                 0.0|\n",
      "|       2| 2020-01-01 00:39:51|  2020-01-01 00:42:54|                 N|       1.0|         134|          28|            1.0|         1.15|        5.5|  0.5|    0.5|       1.0|         0.0|     null|                  0.3|         7.8|         1.0|      1.0|                 0.0|\n",
      "|       1| 2020-01-01 00:00:21|  2020-01-01 00:10:19|                 N|       1.0|          89|          39|            1.0|          2.3|       10.0|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|        11.3|         2.0|      1.0|                 0.0|\n",
      "|       1| 2020-01-01 00:13:59|  2020-01-01 00:21:31|                 N|       1.0|          66|          65|            3.0|          1.0|        6.5|  0.5|    0.5|       0.0|         0.0|     null|                  0.3|         7.8|         2.0|      1.0|                 0.0|\n",
      "+--------+--------------------+---------------------+------------------+----------+------------+------------+---------------+-------------+-----------+-----+-------+----------+------------+---------+---------------------+------------+------------+---------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_green.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95b843e2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}