Fix/3911-cl-store infer storage type from url

codalab/lib/bundle_cli.py

@@ -1054,6 +1054,20 @@ def do_store_command(self, args):
                 "url": args.url,
                 "authentication": args.authentication,
             }
+            if args.url.startswith("azfs://"):
+                if args.storage_type is None:
+                    bundle_store_info["storage_type"] = "azure_blob"
+                elif args.storage_type != "azure_blob":
+                    raise UsageError(
+                        f"cl store storage type '{args.storage_format}' conflicts with storage format."
+                    )
+            elif args.url.startswith("gs://"):
+                if args.storage_type is None:
+                    bundle_store_info["storage_type"] = "gcs"
+                elif args.storage_type != "gcs":
+                    raise UsageError(
+                        f"cl store storage type '{args.storage_format}' conflicts with storage format."
+                    )
             new_bundle_store = client.create('bundle_stores', bundle_store_info)
             print(new_bundle_store["id"], file=self.stdout)
         elif args.command == 'ls':

tests/cli/test_cli.py

@@ -1077,7 +1077,64 @@ def test_upload_default_bundle_store(ctx):
     uuid = _run_command([cl, 'upload', '-c', 'hello'])
     check_contains(bundle_store_name, _run_command([cl, "info", uuid]))
 
+@TestModule.register('store_add')
+def test_store_add(ctx):
+    """
+    Tests of command `cl store add` on different blob storage. (Not test --storage-format yet)
+    1. Specify storage type and url
+    2. Specify url but not specify storage type
+    3. Specify wrong storage type and url: expected exit -1
+    """
+    # Create a new azure_blob bundle store, then delete it
+    bundle_store_name = os.getenv('CODALAB_DEFAULT_BUNDLE_STORE_NAME')
+    blob_id = _run_command(
+        [
+            cl,
+            "store",
+            "add",
+            "--name",
+            bundle_store_name,
+            '--storage-type',
+            'azure_blob',
+            '--url',
+            'azfs://devstoreaccount1/bundles',
+        ]
+    )
+    check_contains("azure_blob", _run_command([cl, "store", "ls"]))
+    _run_command([cl, "store", "rm", blob_id])
+    # TODO: set CODALAB_GOOGLE_APPLICATION_CREDENTIALS and test gcp blob storage
+
+    # create a new azure_blob but not specify storage type
+    blob_id = _run_command(
+        [
+            cl,
+            "store",
+            "add",
+            "--name",
+            bundle_store_name,
+            '--url',
+            'azfs://devstoreaccount1/bundles',
+        ]
+    )
+    check_contains("azure_blob", _run_command([cl, "store", "ls"]))
+    _run_command([cl, "store", "rm", blob_id])
 
+    # create a new azure_blob but specify wrong type
+    blob_id = _run_command(
+        [
+            cl,
+            "store",
+            "add",
+            "--name",
+            bundle_store_name,
+            '--storage-type',
+            'disk',  # the type does not aligns with url
+            '--url',
+            'azfs://devstoreaccount1/bundles',
+        ], 
+        expected_exit_code=1,
+    )
+
 @TestModule.register('download')
 def test_download(ctx):
     # Upload test files directory as archive to preserve everything invariant of the upload implementation