Cloud API: Add software tests for cluster deployment conversation

cratedb_toolkit/util/runtime.py

@@ -46,6 +46,7 @@ def wrapper(*args, **kwargs):
                     if CONFIG.runtime_errors == "raise":
                         raise
                     elif CONFIG.runtime_errors == "exit":  # noqa: RET506
+                        logger.error(ex)
                         sys.exit(runtime_error_exit_code)
                     else:
                         raise NotImplementedError(

tests/cluster/test_examples.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2023, Crate.io Inc.
+# Distributed under the terms of the AGPLv3 license, see LICENSE.
+import responses
+
+import cratedb_toolkit
+
+
+@responses.activate
+def test_example_cloud_cluster_exists(mocker, mock_cloud_cluster_exists):
+    """
+    Verify that the program `examples/cloud_cluster.py` works.
+    In this case, there is a mock which pretends the cluster already exists.
+    """
+
+    cratedb_toolkit.configure(
+        settings_accept_cli=True,
+        settings_accept_env=True,
+    )
+
+    mocker.patch.dict(
+        "os.environ",
+        {
+            "CRATEDB_CLOUD_SUBSCRIPTION_ID": "f33a2f55-17d1-4f21-8130-b6595d7c52db",
+            "CRATEDB_CLOUD_CLUSTER_NAME": "testcluster",
+            "CRATEDB_USERNAME": "crate",
+        },
+    )
+    from examples.cloud_cluster import main
+
+    main()
+
+
+@responses.activate
+def test_example_cloud_cluster_with_deploy(mocker, mock_cloud_cluster_deploy):
+    """
+    Verify that the program `examples/cloud_cluster.py` works.
+    In this case, mocking-wise, there is no cluster, but the test exercises a full cluster deployment.
+    """
+
+    cratedb_toolkit.configure(
+        settings_accept_cli=True,
+        settings_accept_env=True,
+    )
+
+    mocker.patch.dict(
+        "os.environ",
+        {
+            "CRATEDB_CLOUD_SUBSCRIPTION_ID": "f33a2f55-17d1-4f21-8130-b6595d7c52db",
+            "CRATEDB_CLOUD_CLUSTER_NAME": "testcluster",
+            "CRATEDB_USERNAME": "crate",
+        },
+    )
+    from examples.cloud_cluster import main
+
+    main()
+
+
+@responses.activate
+def test_example_cloud_import(mocker, mock_cloud_import):
+    """
+    Verify that the program `examples/cloud_import.py` works.
+    """
+
+    cratedb_toolkit.configure(
+        settings_accept_cli=True,
+        settings_accept_env=True,
+    )
+
+    mocker.patch.dict(
+        "os.environ",
+        {
+            "CRATEDB_CLOUD_CLUSTER_ID": "e1e38d92-a650-48f1-8a70-8133f2d5c400",
+        },
+    )
+    from examples.cloud_import import main
+
+    main()

tests/conftest.py

@@ -1,5 +1,7 @@
 # Copyright (c) 2021-2023, Crate.io Inc.
 # Distributed under the terms of the AGPLv3 license, see LICENSE.
+import json
+
 import pytest
 import responses
 
@@ -86,38 +88,106 @@ def cratedb(cratedb_service):
 
 
 @pytest.fixture
-def cloud_cluster_mock():
+def mock_cloud_cluster_exists(cratedb):
+    """
+    Mock a CrateDB Cloud API conversation, pretending a cluster exists.
+    """
+    responses.add_passthru("http+docker://localhost/")
     responses.add(
-        responses.Response(
-            method="GET",
-            url="https://console.cratedb.cloud/api/v2/clusters/e1e38d92-a650-48f1-8a70-8133f2d5c400/",
-            json={
-                "url": "https://testdrive.example.org:4200/",
+        method="GET",
+        url="https://console.cratedb.cloud/api/v2/clusters/",
+        json=[
+            {
+                "id": "e1e38d92-a650-48f1-8a70-8133f2d5c400",
+                "url": cratedb.get_connection_url(),
                 "project_id": "3b6b7c82-d0ab-458c-ae6f-88f8346765ee",
                 "name": "testcluster",
-            },
-        )
+            }
+        ],
     )
+
+
+@pytest.fixture
+def mock_cloud_cluster_deploy(cratedb):
+    """
+    Mock a CrateDB Cloud API conversation, for exercising a full deployment process.
+    """
+    responses.add_passthru("http+docker://localhost/")
+
+    callcount = 0
+
+    def cluster_list_callback(request):
+        nonlocal callcount
+        callcount += 1
+        headers = {}
+        if callcount == 1:
+            data = []
+        else:
+            data = [
+                {
+                    "id": "e1e38d92-a650-48f1-8a70-8133f2d5c400",
+                    "url": cratedb.get_connection_url(),
+                    "project_id": "3b6b7c82-d0ab-458c-ae6f-88f8346765ee",
+                    "name": "testcluster",
+                }
+            ]
+        return 200, headers, json.dumps(data)
+
+    responses.add_callback(
+        method="GET",
+        url="https://console.cratedb.cloud/api/v2/clusters/",
+        callback=cluster_list_callback,
+    )
+
     responses.add(
-        responses.Response(
-            method="POST",
-            url="https://console.cratedb.cloud/api/v2/clusters/e1e38d92-a650-48f1-8a70-8133f2d5c400/import-jobs/",
-            json={"id": "testdrive-job-id", "status": "REGISTERED"},
-        )
+        method="GET",
+        url="https://console.cratedb.cloud/api/v2/projects/",
+        json=[],
     )
+
     responses.add(
-        responses.Response(
-            method="GET",
-            url="https://console.cratedb.cloud/api/v2/clusters/e1e38d92-a650-48f1-8a70-8133f2d5c400/import-jobs/",
-            json=[
-                {
-                    "id": "testdrive-job-id",
-                    "status": "SUCCEEDED",
-                    "progress": {"message": "Import succeeded"},
-                    "destination": {"table": "basic"},
-                }
-            ],
-        )
+        method="POST",
+        url="https://console.cratedb.cloud/api/v2/projects/",
+        json={"id": "3b6b7c82-d0ab-458c-ae6f-88f8346765ee"},
+    )
+
+    responses.add(
+        method="GET",
+        url="https://console.cratedb.cloud/api/v2/projects/3b6b7c82-d0ab-458c-ae6f-88f8346765ee/",
+        json={},
+    )
+
+
+@pytest.fixture
+def mock_cloud_import():
+    """
+    Mock a CrateDB Cloud API conversation, pretending to run a successful data import.
+    """
+    responses.add(
+        method="GET",
+        url="https://console.cratedb.cloud/api/v2/clusters/e1e38d92-a650-48f1-8a70-8133f2d5c400/",
+        json={
+            "url": "https://testdrive.example.org:4200/",
+            "project_id": "3b6b7c82-d0ab-458c-ae6f-88f8346765ee",
+            "name": "testcluster",
+        },
+    )
+    responses.add(
+        method="POST",
+        url="https://console.cratedb.cloud/api/v2/clusters/e1e38d92-a650-48f1-8a70-8133f2d5c400/import-jobs/",
+        json={"id": "testdrive-job-id", "status": "REGISTERED"},
+    )
+    responses.add(
+        method="GET",
+        url="https://console.cratedb.cloud/api/v2/clusters/e1e38d92-a650-48f1-8a70-8133f2d5c400/import-jobs/",
+        json=[
+            {
+                "id": "testdrive-job-id",
+                "status": "SUCCEEDED",
+                "progress": {"message": "Import succeeded"},
+                "destination": {"table": "basic"},
+            }
+        ],
     )
 
 

tests/io/test_import.py

@@ -52,7 +52,7 @@ def test_import_csv_dask_with_progressbar(cratedb, dummy_csv):
 
 @pytest.mark.skip("Does not work. Q: Why? A: Response mocking? Q: And now? A: Just patch the low-level functions!")
 @responses.activate
-def test_import_cloud_file(tmp_path, caplog, cloud_cluster_mock):
+def test_import_cloud_file(tmp_path, caplog, mock_cloud_import):
     """
     CLI test: Invoke `ctk load table ...` for a CrateDB Cloud Import, from a local file.
     """
@@ -86,7 +86,7 @@ def test_import_cloud_file(tmp_path, caplog, cloud_cluster_mock):
 
 
 @responses.activate
-def test_import_cloud_url(caplog, cloud_cluster_mock):
+def test_import_cloud_url(caplog, mock_cloud_import):
     """
     CLI test: Invoke `ctk load table ...` for a CrateDB Cloud Import, from a URL.
     """

tests/retention/test_examples.py

@@ -1,8 +1,5 @@
 # Copyright (c) 2023, Crate.io Inc.
 # Distributed under the terms of the AGPLv3 license, see LICENSE.
-import responses
-
-import cratedb_toolkit
 
 
 def test_example_edit(store):
@@ -21,21 +18,3 @@ def test_example_retire_cutoff(store):
     from examples.retention_retire_cutoff import main
 
     main(dburi=store.database.dburi)
-
-
-@responses.activate
-def test_example_cloud_import(mocker, store, cloud_cluster_mock):
-    """
-    Verify that the program `examples/cloud_import.py` works.
-    """
-
-    cratedb_toolkit.configure(
-        settings_accept_cli=True,
-        settings_accept_env=True,
-    )
-
-    cluster_id = "e1e38d92-a650-48f1-8a70-8133f2d5c400"
-    mocker.patch.dict("os.environ", {"CRATEDB_CLOUD_CLUSTER_ID": cluster_id})
-    from examples.cloud_import import main
-
-    main()