Run tests against latest dev version (main) using KWOK (#55)

This change allows us to test against the latest development version of
the DBaaS Control Plane using KWOK. This is useful because KWOK gives us
the highest test coverage of all test drivers and allows us to test
features as soon as they have been pushed to main on
nuodb-control-plane, even if they have not been deployed to the test
cluster used by the "External DBaaS" variant.

This change also restores some test coverage that was lost by only
running `terraform validate` on the example configs. If the REST server
is the latest dev version, then we can enable full testing of the
example configs, since all resources and data sources should be
supported.

.circleci/config.yml

@@ -12,29 +12,22 @@ executors:
       image: ubuntu-2204:2023.10.1
     resource_class: medium
 
-jobs:
-  functional_tests:
+commands:
+  run_test:
     parameters:
       test-driver:
         description: |
           The test driver to use.
         type: string
-      nuodb-cp-version:
-        description: |
-          The version of the NuoDB Control Plane to test against.
-        type: string
-        default: 2.5.0
-    executor: ubuntu_vm
-    environment:
-      TEST_RESULTS: /tmp/test-results
-      OUTPUT_DIR: /tmp/test-artifacts
-      NUODB_CP_VERSION: << parameters.nuodb-cp-version >>
     steps:
-      - checkout
+      - run:
+          name: "Set environment variables"
+          command: |
+            echo "export TEST_RESULTS=/tmp/test-results" >> "$BASH_ENV"
+            echo "export OUTPUT_DIR=/tmp/test-artifacts" >> "$BASH_ENV"
       - run:
           name: "Setup environment"
           command: |
-            echo "Setting up << parameters.test-driver >> with NuoDB CP << parameters.nuodb-cp-version >>..."
             make setup-<< parameters.test-driver >>
             make env-<< parameters.test-driver >> >> "$BASH_ENV"
       - run:
@@ -58,6 +51,47 @@ jobs:
           name: "Upload test results"
           path: /tmp/test-results
 
+jobs:
+  functional_tests:
+    parameters:
+      test-driver:
+        description: |
+          The test driver to use.
+        type: string
+      nuodb-cp-version:
+        description: |
+          The version of the NuoDB Control Plane to test against.
+        type: string
+        default: 2.5.0
+    executor: ubuntu_vm
+    environment:
+      NUODB_CP_VERSION: << parameters.nuodb-cp-version >>
+    steps:
+      - checkout
+      - run_test:
+          test-driver: << parameters.test-driver >>
+
+  functional_tests_kwok_dev:
+    executor: ubuntu_vm
+    environment:
+      IMG_REPO: nuodb/nuodb-control-plane
+      IMG_TAG: latest
+      NUODB_CP_IMAGE: nuodb/nuodb-control-plane:latest
+      USING_LATEST_API: "true"
+    steps:
+      - checkout
+      - add_ssh_keys:
+          fingerprints:
+            - "SHA256:b6Diu+dTKXL7N/Y8u3PElZ0sZxk4M+vK9Vb96O6X6ZM"
+      - run:
+          name: "Checkout NuoDB Control Plane repo and build image"
+          command: |
+            git clone [email protected]:nuodb/nuodb-control-plane.git
+            make -C nuodb-control-plane docker-build
+            echo "export NUODB_CP_CRD_CHART=$(pwd)/nuodb-control-plane/charts/nuodb-cp-crd" >> "$BASH_ENV"
+      - run_test:
+          test-driver: kwok
+
   check_quality:
     executor: go
     steps:
@@ -88,6 +122,8 @@ jobs:
 workflows:
   test:
     jobs:
+      - functional_tests_kwok_dev:
+          name: Functional tests (KWOK, dev)
       - functional_tests:
           name: Functional tests (KWOK)
           test-driver: kwok

deploy/kwok/setup.sh

@@ -52,7 +52,11 @@ provisioner: nuodb.github.io/noop-provisioner
 EOF
 
 echo "Installing CRDs for DBaaS..."
-helm install nuodb-cp-crd nuodb-cp-crd --repo "$NUODB_CP_REPO" --version "$NUODB_CP_VERSION"
+if [ -n "$NUODB_CP_CRD_CHART" ]; then
+    helm install nuodb-cp-crd "$NUODB_CP_CRD_CHART"
+else
+    helm install nuodb-cp-crd nuodb-cp-crd --repo "$NUODB_CP_REPO" --version "$NUODB_CP_VERSION"
+fi
 
 # Create service tiers. The Helm features do not matter, since they are not
 # exposed in Terraform.

docs/data-sources/backuppolicy.md

@@ -17,6 +17,7 @@ Data source for exposing information about NuoDB backup policies created using t
 data "nuodbaas_backuppolicy" "policy_details" {
   organization = "org"
   name         = "pol"
+  depends_on   = [nuodbaas_backuppolicy.pol]
 }
 ```
 

docs/data-sources/database.md

@@ -18,6 +18,7 @@ data "nuodbaas_database" "database_details" {
   organization = "org"
   project      = "proj"
   name         = "db"
+  depends_on   = [nuodbaas_database.db]
 }
 ```
 

docs/data-sources/project.md

@@ -17,6 +17,7 @@ Data source for exposing information about NuoDB projects created using the DBaa
 data "nuodbaas_project" "project_details" {
   organization = "org"
   name         = "proj"
+  depends_on   = [nuodbaas_project.proj]
 }
 ```
 

examples/data-sources/nuodbaas_backuppolicy/data-source.tf

@@ -2,4 +2,5 @@
 data "nuodbaas_backuppolicy" "policy_details" {
   organization = "org"
   name         = "pol"
+  depends_on   = [nuodbaas_backuppolicy.pol]
 }

examples/data-sources/nuodbaas_database/data-source.tf

@@ -3,4 +3,5 @@ data "nuodbaas_database" "database_details" {
   organization = "org"
   project      = "proj"
   name         = "db"
+  depends_on   = [nuodbaas_database.db]
 }

examples/data-sources/nuodbaas_project/data-source.tf

@@ -2,4 +2,5 @@
 data "nuodbaas_project" "project_details" {
   organization = "org"
   name         = "proj"
+  depends_on   = [nuodbaas_project.proj]
 }

internal/provider_test/examples_test.go

@@ -18,6 +18,10 @@ import (
 	. "github.com/nuodb/terraform-provider-nuodbaas/internal/provider"
 )
 
+const (
+	USING_LATEST_API TestOption = "USING_LATEST_API"
+)
+
 func CombineConfigs(t *testing.T, root string) string {
 	var combinedConfigs string
 	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
@@ -33,7 +37,7 @@ func CombineConfigs(t *testing.T, root string) string {
 	return combinedConfigs
 }
 
-func TestValidateExamples(t *testing.T) {
+func TestExamples(t *testing.T) {
 	// Create provider server that runs within test
 	ctx := context.Background()
 	reattachCfg, closeFn := CreateProviderServer(t, ctx)
@@ -57,7 +61,20 @@ func TestValidateExamples(t *testing.T) {
 	_, err = tf.Init()
 	require.NoError(t, err)
 
-	// Validate config
-	_, err = tf.Validate()
-	require.NoError(t, err)
+	if USING_LATEST_API.IsTrue() {
+		// Run `terraform apply` on config
+		_, err = tf.Apply()
+		defer tf.DestroySilently()
+		require.NoError(t, err)
+
+		// Run `terraform refresh` to update data sources
+		_, err = tf.Run("refresh")
+		require.NoError(t, err)
+	} else {
+		// The server may not support all resources and data sources, if
+		// it is running at all. Run `terraform validate` only, which
+		// does not require a server connection.
+		_, err = tf.Validate()
+		require.NoError(t, err)
+	}
 }