Fix Solr Auth Binding

.github/workflows/test.yml

@@ -38,7 +38,7 @@ jobs:
         fetch-depth: '0'
 
     - name: Build the brokerpak and set up the test environment
-      run: make build test-env-up 
+      run: make build kind-up 
 
     - name: Stop for debugging
       if: github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled
@@ -47,7 +47,7 @@ jobs:
         limit-access-to-actor: true  
 
     - name: Start the broker and run the tests
-      run: make up test
+      run: make .env up test
 
     - name: Clean up if there was a failure
       if: ${{ failure() }}

.gitignore

@@ -50,3 +50,4 @@ examples.json
 tee
 .ash_history
 .terraform.d/
+terraform/provision/.cache

Makefile

@@ -23,7 +23,7 @@ check:
 clean: SHELL:=./test_env_load
 clean: down ## Bring down the broker service if it's up and clean out the database
 	@docker rm -f csb-service-$${SERVICE_NAME}
-	@rm datagov-services-pak-*.brokerpak
+	@rm -f datagov-services-pak-*.brokerpak
 
 # Origin of the subdirectory dependency solution: 
 # https://stackoverflow.com/questions/14289513/makefile-rule-that-depends-on-all-files-under-a-directory-including-within-subd#comment19860124_14289872
@@ -84,7 +84,7 @@ demo-down: examples.json ## Clean up data left over from tests and demos
 	-@helm uninstall example 2>/dev/null ; true
 	-@kubectl delete secret basic-auth1 2>/dev/null ; true
 
-test-env-up: ## Set up a Kubernetes test environment using KinD
+kind-up: ## Set up a Kubernetes test environment using KinD
 	# Creating a temporary Kubernetes cluster to test against with KinD
 	@kind create cluster --config kind-config.yaml --name datagov-broker-test
 	# Grant cluster-admin permissions to the `system:serviceaccount:default:default` Service.
@@ -99,20 +99,19 @@ test-env-up: ## Set up a Kubernetes test environment using KinD
       --selector=app.kubernetes.io/component=controller \
       --timeout=270s
 	# Install the ZooKeeper and Solr operators using Helm
-	# TODO: Update the CRD installation in the eks-brokerpak as well
-	kubectl create -f https://solr.apache.org/operator/downloads/crds/v0.4.0/all-with-dependencies.yaml
-	@helm install --namespace kube-system --repo https://solr.apache.org/charts --version 0.4.0 solr solr-operator
+	kubectl create -f https://solr.apache.org/operator/downloads/crds/v0.5.0/all-with-dependencies.yaml
+	@helm install --namespace kube-system --repo https://solr.apache.org/charts --version 0.5.0 solr solr-operator
 
-.env: $(HOME)/.kube/config generate-env.sh
-	@echo Generating a .env file containing k8s config for the broker
-	@./generate-env.sh > .env
+.env: generate-env.sh
+	@echo Generating a .env file containing the k8s config needed by the broker
+	@./generate-env.sh
 
-test-env-down: ## Tear down the Kubernetes test environment in KinD
+kind-down: ## Tear down the Kubernetes test environment in KinD
 	kind delete cluster --name datagov-broker-test
 	@rm .env
 
-all: clean build test-env-up up test down test-env-down ## Clean and rebuild, start test environment, run the broker, run the examples, and tear the broker and test env down
-.PHONY: all clean build up down test test-env-up test-env-down
+all: clean build kind-up up test down kind-down ## Clean and rebuild, start local test environment, run the broker, run the examples, and tear the broker and test env down
+.PHONY: all clean build up down test kind-up kind-down
 
 # Output documentation for top-level targets
 # Thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html

README.md

@@ -19,6 +19,13 @@ docs.
 Huge props go to @josephlewis42 of Google for publishing and publicizing the
 brokerpak concept, and to the Pivotal team running with the concept!
 
+### Related Docs
+
+- [Solr Helm Chart](https://artifacthub.io/packages/helm/apache-solr/solr)
+- [Solr Operator Helm Chart](https://artifacthub.io/packages/helm/apache-solr/solr-operator)
+- [Solr Official Docs](https://solr.apache.org/guide/8_11/)
+- [Solr Operator SolrCloud CRD](https://github.com/apache/solr-operator/blob/main/docs/solr-cloud/solr-cloud-crd.md)
+
 ## Prerequisites
 
 1. `make` is used for executing docker commands in a meaningful build cycle.
@@ -43,42 +50,54 @@ down       Bring the cloud-service-broker service down
 test       Execute the brokerpak examples against the running broker
 demo-up    Provision a SolrCloud instance and output the bound credentials
 demo-down  Clean up data left over from tests and demos
-test-env-up Set up a Kubernetes test environment using KinD
-test-env-down Tear down the Kubernetes test environment in KinD
+kind-up    Set up a local Kubernetes test environment using KinD
+kind-down  Tear down the Kubernetes test environment in KinD
 all        Clean and rebuild, start test environment, run the broker, run the examples, and tear the broker and test env down
 help       This help
 ```
 
 Notable targets are described below.
 
-## Operating a test/demo Kubernetes environment
+## Providing a test/demo Kubernetes environment
+
+To use an existing Kubernetes cluster for testing:
+
+- Ensure that the [solr-operator Helm chart](https://artifacthub.io/packages/helm/apache-solr/solr-operator) is installed (at least 0.5.0)
+- Ensure that the [ingress-nginx Helm chart](https://artifacthub.io/packages/helm/ingress-nginx/ingress-nginx) is installed
+- Set the `SOLR_DOMAIN_NAME` environment variable to the subdomain where `ingress-nginx` resources will be mapped
+- Set the `KUBECONFIG` environment variable to point to the kubeconfig file for the cluster
+- If your kubeconfig describes multiple clusters, make sure the current cluster is set to the right one
 
-### Creating the environment
+If you don't have an existing Kubernetes cluster, you can create a local test environment in Docker using the Makefile.
+
+### Creating a local k8s environment for testing
 
 Create a temporary Kubernetes cluster to test against with KinD:
 
 ```bash
-make test-env-up
+make kind-up
 ```
 
-### Tearing down the environment
+### Tearing down the local k8s environment
 
-Run 
+Run
 
 ```bash
-make test-env-down
+make kind-down
 ```
 
 ## Iterating on the Terraform code
 
-To work with the Terraform and KinD cluster directly (eg not through the CSB or brokerpak), you can generate an appropriate .tfvars file by running:
+To work with the Terraform and target cluster directly (eg not through the CSB or brokerpak), you can generate an appropriate .tfvars file by running:
 
 ```bash
 make .env
 ```
 
 From that point on, you can `cd terraform/provision` and iterate with `terraform init/plan/apply/etc`. The same configuration is also available in `terraform/bind`.
 
+(Note if you've been working with the broker the configuration will probably already exist.)
+
 ## Building and starting the brokerpak (while the test environment is available)
 
 Run

examples.json-template

@@ -5,7 +5,8 @@
     "service_name": "solr-cloud",
     "service_id": "b9013a91-9ce8-4c18-8035-a135a8cd6ff9",
     "plan_id": "e35e9675-413f-4f42-83de-ad5003357e77",
-    "provision_params": {"solrJavaMem":"-Xms300m -Xmx300m", "solrMem":"1G", "solrCpu":"1000m", "cloud_name":"demo"},
+    "provision_params": {"solrJavaMem":"-Xms300m -Xmx300m", "solrMem":"1G", "solrCpu":"1000m", "cloud_name":"demo", "solrImageRepo":"ghcr.io/gsa/catalog.data.gov.solr", "solrImageTag":"8-curl"},
     "bind_params": {}
   }
-]
+]
+

generate-env.sh

@@ -3,33 +3,48 @@
 set -e
 
 CURRENT_CONTEXT=$(kubectl config current-context)
-SOLR_CLUSTER_CA_CERTIFICATE=$(kubectl config view --raw -o json | jq -r '.clusters[]| select(.name | contains("'${CURRENT_CONTEXT}'"))  .cluster["certificate-authority-data"]')
-SOLR_TOKEN=$(kubectl get secret $( kubectl get serviceaccount default -n default -o json | jq -r '.secrets[0].name' ) -n default -o json | jq -r .data.token)
+CURRENT_CLUSTER=$(kubectl config view --raw -o json | jq -r '.contexts[]| select(.name | contains("'"${CURRENT_CONTEXT}"'"))  .context.cluster')
+CURRENT_USER=$(kubectl config view --raw -o json | jq -r '.contexts[]| select(.name | contains("'"${CURRENT_CONTEXT}"'"))  .context.user')
+SOLR_CLUSTER_CA_CERTIFICATE=$(kubectl config view --raw -o json | jq -r '.clusters[]| select(.name | contains("'"${CURRENT_CLUSTER}"'"))  .cluster["certificate-authority-data"]')
+SOLR_TOKEN=$(echo -n `kubectl config view --raw -o json | jq -r '.users[]| select(.name | contains("'"${CURRENT_USER}"'"))  .user["token"]'` | base64 -w 0)
+SOLR_SERVER=$(kubectl config view --raw -o json | jq -r '.clusters[]| select(.name | contains("'"${CURRENT_CLUSTER}"'"))  .cluster["server"]')
 
-# We need the Docker-internal control plane URL to be resolved for the CSB
-# when running in a container
-SOLR_DOCKER_SERVER=$(kind get kubeconfig --internal --name=$(kind get clusters | grep datagov-broker-test) | grep server | cut -d ' ' -f 6-)
+SOLR_DOMAIN_NAME=${SOLR_DOMAIN_NAME:-ing.local.domain}
 
-# We need the localhost control plan URL to be used for direct access when we
-# work outside the CSB
-SOLR_LOCALHOST_SERVER=$(kind get kubeconfig --name=$(kind get clusters | grep datagov-broker-test) | grep server | cut -d ' ' -f 6-)
+if [[ "${CURRENT_CLUSTER}" == "kind-datagov-broker-test" ]]; then
+    # If the test cluster is in KinD we need the CSB to use 
+    # a control plane URL resolvable from inside the CSB Docker container
+    CURRENT_USER=kind-datagov-broker-test
+    SOLR_CP_SERVER=$(kind get kubeconfig --internal --name="$(kind get clusters | grep datagov-broker-test)" | grep server | cut -d ' ' -f 6-)
+    SOLR_TOKEN=$(kubectl get secret $(kubectl get secrets | grep -oh "default-token-[a-z]*\s") -o json | jq .data.token | tr -d '"')
+    if [[ "$SOLR_TOKEN" == "null" ]]; then
+        # The format of the secret is different if there are more than one token associated with a secret.
+        # The first token works reliably
+        SOLR_TOKEN=$(kubectl get secret $(kubectl get secrets | grep -oh "default-token-[a-z]*\s") -o json | jq .items[0].data.token | tr -d '"')
+    fi
+else
+    # Otherwise it's the same as the normal server control plane URL
+    SOLR_CP_SERVER=${SOLR_SERVER}
+fi
 
 # Generate the environment variables needed for configuring the CSB running in Docker
-echo SOLR_SERVER=${SOLR_DOCKER_SERVER} > .env
-echo SOLR_TOKEN=${SOLR_TOKEN} >> .env
-echo SOLR_CLUSTER_CA_CERTIFICATE=${SOLR_CLUSTER_CA_CERTIFICATE} >> .env
-echo SOLR_NAMESPACE=default >> .env
-echo SOLR_DOMAIN_NAME=ing.local.domain >> .env
+cat > .env << HEREDOC
+SOLR_SERVER=${SOLR_CP_SERVER}
+SOLR_TOKEN=${SOLR_TOKEN}
+SOLR_CLUSTER_CA_CERTIFICATE=${SOLR_CLUSTER_CA_CERTIFICATE}
+SOLR_NAMESPACE=default
+SOLR_DOMAIN_NAME=${SOLR_DOMAIN_NAME}
+HEREDOC
 
 # Generate terraform.tfvars needed for mucking about directly with terraform/provision
 cat > terraform/provision/terraform.tfvars << HEREDOC
-server="${SOLR_LOCALHOST_SERVER}"
+server="${SOLR_CP_SERVER}"
 token="${SOLR_TOKEN}"
 cluster_ca_certificate="${SOLR_CLUSTER_CA_CERTIFICATE}"
 namespace="default"
-domain_name="ing.local.domain"
+domain_name="${SOLR_DOMAIN_NAME}"
 replicas=3
-solrImageTag="8.6"
+solrImageTag="8.11"
 solrJavaMem="-Xms300m -Xmx300m"
 cloud_name="example"
 solrCpu="1000m"

generate-examples.sh

@@ -7,8 +7,8 @@ CURRENT_CONTEXT=$(kubectl config current-context)
 CLUSTER_CA_CERTIFICATE=$(kubectl config view --raw -o json | jq -r '.clusters[]| select(.name | contains("'${CURRENT_CONTEXT}'"))  .cluster["certificate-authority-data"]')
 TOKEN=$(kubectl get secret $( kubectl get serviceaccount default -n default -o json | jq -r '.secrets[0].name' ) -n default -o json | jq -r .data.token)
 
-# We need the Docker-internal control plane URL to be resolved
-SERVER=$(kind get kubeconfig --internal --name=$(kind get clusters | grep datagov-broker-test) | grep server | cut -d ' ' -f 6-)
+CURRENT_CLUSTER=$(kubectl config view --raw -o json | jq -r '.contexts[]| select(.name | contains("'${CURRENT_CONTEXT}'"))  .context.cluster')
+SERVER=$(kubectl config view --raw -o json | jq -r '.clusters[]| select(.name | contains("'${CURRENT_CLUSTER}'"))  .cluster["server"]')
 
 template() {
   file=examples.json-template

solr-cloud.yml

@@ -29,7 +29,7 @@ provision:
   - field_name: solrImageTag
     required: false
     type: string
-    details: "Tag for the Solr Docker image to use, defaults to 8.6. See https://hub.docker.com/_/solr?tab=tags (or your configured solrImageRepo) for options"
+    details: "Tag for the Solr Docker image to use, defaults to 8.11. See https://hub.docker.com/_/solr?tab=tags (or your configured solrImageRepo) for options"
   - field_name: solrJavaMem
     required: false
     type: string
@@ -83,7 +83,7 @@ provision:
     overwrite: false
     type: string
   - name: solrImageTag
-    default: "6.6.6-slim"
+    default: "8.11-slim"
     overwrite: false
     type: string
   - name: solrJavaMem
@@ -101,7 +101,6 @@ provision:
   outputs: []
   template_refs: 
     main: terraform/provision/main.tf
-    data: terraform/provision/data.tf
     outputs: terraform/provision/outputs.tf
     providers: terraform/provision/providers.tf
     variables: terraform/provision/variables.tf

terraform/bind/data.tf

@@ -1,9 +1,3 @@
-data "kubernetes_namespace" "namespace" {
-  metadata {
-    name = var.namespace
-  }
-}
-
 # Derive the ingress hostname that's used for connecting to the exposed SolrCloud
 data "kubernetes_ingress" "solrcloud-ingress" {
   metadata {

terraform/bind/main.tf

@@ -70,23 +70,20 @@ resource "null_resource" "manage_solr_user" {
     # Can't reuse containers because they are left in an unpredictable state after a single run
     # Wait for the command to run before deleting the container
     command = <<-EOF
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) run temp1 --image=curlimages/curl --  \
+      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) exec ${self.triggers.cloud_name}-solrcloud-0 --  curl \
         -s -f -L \
         -o /dev/null \
         -w "%%{http_code}\n" \
         --user admin:$${ADMIN_PASSWORD} \
         'http://${self.triggers.cloud_name}-solrcloud-common/solr/admin/authentication' \
         -H 'Content-type:application/json' --data "$CREATE_USER_JSON"
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) run temp2 --image=curlimages/curl --  \
+      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) exec ${self.triggers.cloud_name}-solrcloud-0 --  curl \
         -s -f -L \
         -o /dev/null \
         -w "%%{http_code}\n" \
         --user admin:$${ADMIN_PASSWORD} \
         'http://${self.triggers.cloud_name}-solrcloud-common/solr/admin/authorization' \
         -H 'Content-type:application/json' --data "$SET_ROLE_JSON"
-      sleep 10
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) delete pod temp2
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) delete pod temp1
     EOF
   }
 
@@ -103,23 +100,20 @@ resource "null_resource" "manage_solr_user" {
     # Can't reuse containers because they are left in an unpredictable state after a single run
     # Wait for the command to run before deleting the container
     command = <<-EOF
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) run temp1 --image=curlimages/curl --  \
+      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d)  exec ${self.triggers.cloud_name}-solrcloud-0 --  curl \
         -s -f -L \
         -o /dev/null \
         -w "%%{http_code}\n" \
         --user admin:$ADMIN_PASSWORD \
         'http://${self.triggers.cloud_name}-solrcloud-common/solr/admin/authorization' \
         -H 'Content-type:application/json' --data "$CLEAR_ROLE_JSON"
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) run temp2 --image=curlimages/curl --  \
+      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d)  exec ${self.triggers.cloud_name}-solrcloud-0 --  curl \
         -s -f -L \
         -o /dev/null \
         -w "%%{http_code}\n" \
         --user admin:$ADMIN_PASSWORD \
         'http://${self.triggers.cloud_name}-solrcloud-common/solr/admin/authentication' \
         -H 'Content-type:application/json' --data "$DELETE_USER_JSON"
-      sleep 10
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) delete pod temp2
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) delete pod temp1
     EOF
   }
 

terraform/provision/README.md

@@ -1,6 +1,6 @@
 # How to iterate on the provisioning code
 
-**WARNING: Communication to the KIND Cluster doesn't currently exist from this
+**WARNING: Communication to the KinD Cluster doesn't currently exist from this
 container.  This needs to be established before terraform can work in this 
 container.**
 

terraform/provision/main.tf

@@ -26,7 +26,7 @@ resource "helm_release" "solrcloud" {
   name            = local.cloud_name
   chart           = "solr"
   repository      = "https://solr.apache.org/charts"
-  namespace       = data.kubernetes_namespace.namespace.id
+  namespace       = var.namespace
   cleanup_on_fail = true
   atomic          = true
   wait            = true
@@ -45,6 +45,7 @@ resource "helm_release" "solrcloud" {
       "replicas"                                   = var.replicas      # How many replicas you want
       "solrOptions.javaMemory"                     = var.solrJavaMem   # How much memory to give each replica
       "solrOptions.security.authenticationType"    = "Basic"
+      "ingressOptions.annotations.nginx\\.ingress\\.kubernetes\\.io/proxy-body-size" = "999m"
     }
     content {
       name  = set.key
@@ -64,7 +65,7 @@ resource "helm_release" "solrcloud" {
     }
     command = <<-EOF
       sleep 30
-      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) wait --for=condition=ready --timeout=3600s -n ${data.kubernetes_namespace.namespace.id} pod -l solr-cloud=${local.cloud_name}
+      kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) wait --for=condition=ready --timeout=3600s -n ${var.namespace} pod -l solr-cloud=${local.cloud_name}
     EOF
   }