Add CLI functions + minor changes

skylark/cli/cli.py

@@ -78,6 +78,10 @@ def cp(src: str, dst: str):
         copy_local_s3(Path(path_src), bucket_dst, path_dst)
     elif provider_src == "s3" and provider_dst == "local":
         copy_s3_local(bucket_src, path_src, Path(path_dst))
+    elif provider_src == "local" and provider_dst == "gcs":
+        copy_local_gcs(Path(path_src), bucket_dst, path_dst)
+    elif provider_src == "gcs" and provider_dst == "local":
+        copy_gcs_local(bucket_src, path_src, Path(path_dst))
     else:
         raise NotImplementedError(f"{provider_src} to {provider_dst} not supported yet")
 

skylark/cli/cli_helper.py

@@ -84,6 +84,11 @@ def copy_local_local(src: Path, dst: Path):
         dst.parent.mkdir(exist_ok=True, parents=True)
         copyfile(src, dst)
 
+def copy_local_gcs(src: Path, dst_bucket: str, dst_key: str):
+    raise NotImplementedError(f"GCS not yet supported")
+
+def copy_gcs_local(src_bucket: str, src_key: str, dst: Path):
+    raise NotImplementedError(f"GCS not yet supported")
 
 def copy_local_s3(src: Path, dst_bucket: str, dst_key: str, use_tls: bool = True):
     s3 = S3Interface(None, dst_bucket, use_tls=use_tls)

skylark/obj_store/gcs_interface.py

@@ -10,17 +10,14 @@
 
 
 class GCSInterface(ObjectStoreInterface):
-    def __init__(self, gcp_region, bucket_name, use_tls=True, part_size=None, throughput_target_gbps=None):
+    def __init__(self, gcp_region, bucket_name):
         # TODO: infer region?
         self.gcp_region = gcp_region
 
         self.bucket_name = bucket_name
         self.pending_downloads, self.completed_downloads = 0, 0
         self.pending_uploads, self.completed_uploads = 0, 0
 
-        self.gcs_part_size = part_size
-        self.gcs_throughput_target_gbps = throughput_target_gbps
-
         # TODO - figure out how paralllelism handled
         self._gcs_client = storage.Client()
 
@@ -43,11 +40,10 @@ def bucket_exists(self):
             print(e)
             return False
 
-    def create_bucket(self):
+    def create_bucket(self, storage_class: str = "STANDARD"):
         if not self.bucket_exists():
             bucket = self._gcs_client.bucket(self.bucket_name)
-            bucket.storage_class = "COLDLINE"  # TODO: which storage class?
-            print(self.gcp_region)
+            bucket.storage_class = storage_class
             new_bucket = self._gcs_client.create_bucket(bucket, location=self.gcp_region)
         assert self.bucket_exists()