allow https url for spark ingestion jar (#1266)

* allow https url for spark ingestion jar Signed-off-by: Jacob Klegar <[email protected]> * lint Signed-off-by: Jacob Klegar <[email protected]>
feast-dev · Jan 18, 2021 · 7561970 · 7561970
1 parent cd57909
commit 7561970
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 7 deletions.
diff --git a/sdk/python/feast/pyspark/launchers/aws/emr_utils.py b/sdk/python/feast/pyspark/launchers/aws/emr_utils.py
@@ -80,13 +80,13 @@ def _random_string(length) -> str:
     return "".join(random.choice(string.ascii_lowercase) for _ in range(length))
 
 
-def _upload_jar(jar_s3_prefix: str, local_path: str) -> str:
-    with open(local_path, "rb") as f:
-        uri = urlparse(os.path.join(jar_s3_prefix, os.path.basename(local_path)))
+def _upload_jar(jar_s3_prefix: str, jar_path: str) -> str:
+    if jar_path.startswith("https://"):
+        return jar_path
+    with open(jar_path, "rb") as f:
+        uri = urlparse(os.path.join(jar_s3_prefix, os.path.basename(jar_path)))
         return urlunparse(
-            get_staging_client(uri.scheme).upload_fileobj(
-                f, local_path, remote_uri=uri,
-            )
+            get_staging_client(uri.scheme).upload_fileobj(f, jar_path, remote_uri=uri)
         )
 
 

diff --git a/sdk/python/feast/pyspark/launchers/k8s/k8s.py b/sdk/python/feast/pyspark/launchers/k8s/k8s.py
@@ -276,7 +276,11 @@ def historical_feature_retrieval(
         return cast(RetrievalJob, self._job_from_job_info(job_info))
 
     def _upload_jar(self, jar_path: str) -> str:
-        if jar_path.startswith("s3://") or jar_path.startswith("s3a://"):
+        if (
+            jar_path.startswith("s3://")
+            or jar_path.startswith("s3a://")
+            or jar_path.startswith("https://")
+        ):
             return jar_path
         elif jar_path.startswith("file://"):
             local_jar_path = urlparse(jar_path).path