HCP Observability acceptance test

acceptance/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/hashicorp/consul/sdk v0.14.0-rc1
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/hashicorp/go-version v1.6.0
+	github.com/hashicorp/hcp-sdk-go v0.50.0
 	github.com/hashicorp/serf v0.10.1
 	github.com/hashicorp/vault/api v1.8.3
 	github.com/stretchr/testify v1.8.3
@@ -24,6 +25,7 @@ require (
 require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/aws/aws-sdk-go v1.44.262 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
@@ -39,9 +41,18 @@ require (
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/analysis v0.21.4 // indirect
+	github.com/go-openapi/errors v0.20.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/loads v0.21.2 // indirect
+	github.com/go-openapi/runtime v0.25.0 // indirect
+	github.com/go-openapi/spec v0.20.8 // indirect
+	github.com/go-openapi/strfmt v0.21.3 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/validate v0.22.1 // indirect
+	github.com/go-ozzo/ozzo-validation v3.6.0+incompatible // indirect
 	github.com/go-sql-driver/mysql v1.5.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -77,6 +88,7 @@ require (
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
 	github.com/miekg/dns v1.1.50 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.0 // indirect
@@ -88,6 +100,8 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/run v1.0.0 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -98,14 +112,18 @@ require (
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/urfave/cli v1.22.2 // indirect
+	go.mongodb.org/mongo-driver v1.11.0 // indirect
+	go.opentelemetry.io/otel v1.11.1 // indirect
+	go.opentelemetry.io/otel/trace v1.11.1 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
 	golang.org/x/mod v0.9.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
+	golang.org/x/oauth2 v0.6.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
@@ -119,7 +137,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/component-base v0.26.3 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

acceptance/tests/cloud/basic_test.go

@@ -150,6 +150,7 @@ func TestBasicCloud(t *testing.T) {
 	releaseName := helpers.RandomName()
 
 	helmValues := map[string]string{
+		"global.imagePullPolicy":             "IfNotPresent",
 		"global.cloud.enabled":               "true",
 		"global.cloud.resourceId.secretName": resourceSecretName,
 		"global.cloud.resourceId.secretKey":  resourceSecretKey,

acceptance/tests/cloud/remote_dev_test.go

@@ -0,0 +1,264 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package cloud
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+
+	hcpgnm "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/client/global_network_manager_service"
+	"github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/models"
+	hcpcfg "github.com/hashicorp/hcp-sdk-go/config"
+	"github.com/hashicorp/hcp-sdk-go/httpclient"
+	"github.com/hashicorp/hcp-sdk-go/resource"
+)
+
+type DevTokenResponse struct {
+	Token string `json:"token"`
+}
+
+type hcp struct {
+	ResourceID   string
+	ClientID     string
+	ClientSecret string
+	AuthURL      string
+	APIHostname  string
+	ScadaAddress string
+}
+
+func TestRemoteDevCloud(t *testing.T) {
+	_, rIDok := os.LookupEnv("HCP_RESOURCE_ID")
+	_, cIDok := os.LookupEnv("HCP_CLIENT_ID")
+	_, cSECok := os.LookupEnv("HCP_CLIENT_SECRET")
+
+	if !rIDok || !cIDok || !cSECok {
+		t.Log("Must set HCP_RESOURCE_ID, HCP_CLIENT_ID and HCP_CLIENT_SECRET")
+		t.FailNow()
+	}
+
+	apiHost := os.Getenv("HCP_AUTH_URL")
+	if apiHost == "" {
+		apiHost = "https://api.hcp.dev"
+	}
+	authURL := os.Getenv("HCP_API_HOST")
+	if authURL == "" {
+		authURL = "https://auth.idp.hcp.dev"
+	}
+	scadaAddr := os.Getenv("HCP_SCADA_ADDRESS")
+	if scadaAddr == "" {
+		scadaAddr = "scada.internal.hcp.dev:7224"
+	}
+
+	ctx := suite.Environment().DefaultContext(t)
+
+	kubectlOptions := ctx.KubectlOptions(t)
+	ns := kubectlOptions.Namespace
+	k8sClient := environment.KubernetesClientFromOptions(t, kubectlOptions)
+
+	var (
+		resourceSecretName     = "resource-sec-name"
+		resourceSecretKey      = "resource-sec-key"
+		resourceSecretKeyValue = os.Getenv("HCP_RESOURCE_ID")
+
+		clientIDSecretName     = "clientid-sec-name"
+		clientIDSecretKey      = "clientid-sec-key"
+		clientIDSecretKeyValue = os.Getenv("HCP_CLIENT_ID")
+
+		clientSecretName     = "client-sec-name"
+		clientSecretKey      = "client-sec-key"
+		clientSecretKeyValue = os.Getenv("HCP_CLIENT_SECRET")
+
+		apiHostSecretName     = "apihost-sec-name"
+		apiHostSecretKey      = "apihost-sec-key"
+		apiHostSecretKeyValue = apiHost
+
+		authUrlSecretName     = "authurl-sec-name"
+		authUrlSecretKey      = "authurl-sec-key"
+		authUrlSecretKeyValue = authURL
+
+		scadaAddressSecretName     = "scadaaddress-sec-name"
+		scadaAddressSecretKey      = "scadaaddress-sec-key"
+		scadaAddressSecretKeyValue = scadaAddr
+
+		bootstrapTokenSecretName = "bootstrap-token"
+		bootstrapTokenSecretKey  = "token"
+	)
+
+	hcpCfg := hcp{
+		ResourceID:   resourceSecretKeyValue,
+		ClientID:     clientIDSecretKeyValue,
+		ClientSecret: clientSecretKeyValue,
+		AuthURL:      authUrlSecretKeyValue,
+		APIHostname:  apiHostSecretKeyValue,
+		ScadaAddress: scadaAddressSecretKeyValue,
+	}
+
+	aclToken := hcpCfg.fetchAgentBootstrapConfig(t)
+
+	cfg := suite.Config()
+	consul.CreateK8sSecret(t, k8sClient, cfg, ns, resourceSecretName, resourceSecretKey, resourceSecretKeyValue)
+	consul.CreateK8sSecret(t, k8sClient, cfg, ns, clientIDSecretName, clientIDSecretKey, clientIDSecretKeyValue)
+	consul.CreateK8sSecret(t, k8sClient, cfg, ns, clientSecretName, clientSecretKey, clientSecretKeyValue)
+	consul.CreateK8sSecret(t, k8sClient, cfg, ns, apiHostSecretName, apiHostSecretKey, apiHostSecretKeyValue)
+	consul.CreateK8sSecret(t, k8sClient, cfg, ns, authUrlSecretName, authUrlSecretKey, authUrlSecretKeyValue)
+	consul.CreateK8sSecret(t, k8sClient, cfg, ns, scadaAddressSecretName, scadaAddressSecretKey, scadaAddressSecretKeyValue)
+	consul.CreateK8sSecret(t, k8sClient, cfg, ns, bootstrapTokenSecretName, bootstrapTokenSecretKey, aclToken)
+
+	releaseName := helpers.RandomName()
+
+	helmValues := map[string]string{
+		"global.imagePullPolicy":             "IfNotPresent",
+		"global.cloud.enabled":               "true",
+		"global.cloud.resourceId.secretName": resourceSecretName,
+		"global.cloud.resourceId.secretKey":  resourceSecretKey,
+
+		"global.cloud.clientId.secretName": clientIDSecretName,
+		"global.cloud.clientId.secretKey":  clientIDSecretKey,
+
+		"global.cloud.clientSecret.secretName": clientSecretName,
+		"global.cloud.clientSecret.secretKey":  clientSecretKey,
+
+		"global.cloud.apiHost.secretName": apiHostSecretName,
+		"global.cloud.apiHost.secretKey":  apiHostSecretKey,
+
+		"global.cloud.authUrl.secretName": authUrlSecretName,
+		"global.cloud.authUrl.secretKey":  authUrlSecretKey,
+
+		"global.cloud.scadaAddress.secretName": scadaAddressSecretName,
+		"global.cloud.scadaAddress.secretKey":  scadaAddressSecretKey,
+		"connectInject.default":                "true",
+
+		"global.acls.manageSystemACLs":          "true",
+		"global.acls.bootstrapToken.secretName": bootstrapTokenSecretName,
+		"global.acls.bootstrapToken.secretKey":  bootstrapTokenSecretKey,
+
+		"global.gossipEncryption.autoGenerate": "false",
+		"global.tls.enabled":                   "true",
+		"global.tls.enableAutoEncrypt":         "true",
+
+		"telemetryCollector.enabled":                   "true",
+		"telemetryCollector.cloud.clientId.secretName": clientIDSecretName,
+		"telemetryCollector.cloud.clientId.secretKey":  clientIDSecretKey,
+
+		"telemetryCollector.cloud.clientSecret.secretName": clientSecretName,
+		"telemetryCollector.cloud.clientSecret.secretKey":  clientSecretKey,
+		// Either we set the global.trustedCAs (make sure it's idented exactly) or we
+		// set TLS to insecure
+
+		"telemetryCollector.extraEnvironmentVars.HCP_API_ADDRESS": apiHostSecretKeyValue,
+	}
+
+	if cfg.ConsulImage != "" {
+		helmValues["global.image"] = cfg.ConsulImage
+	}
+	if cfg.ConsulCollectorImage != "" {
+		helmValues["telemetryCollector.image"] = cfg.ConsulCollectorImage
+	}
+
+	consulCluster := consul.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), cfg, releaseName)
+	consulCluster.Create(t)
+
+	logger.Log(t, "setting acl permissions for collector and services")
+	aclDir := "../fixtures/bases/cloud/service-intentions"
+	k8s.KubectlApplyK(t, ctx.KubectlOptions(t), aclDir)
+	helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() {
+		k8s.KubectlDeleteK(t, ctx.KubectlOptions(t), aclDir)
+	})
+
+	logger.Log(t, "creating static-server deployment")
+	k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server")
+	time.Sleep(1 * time.Hour)
+
+	// TODO: add in test assertions here
+
+}
+
+// fetchAgentBootstrapConfig use the resource-id, client-id, and client-secret
+// to call to the agent bootstrap config endpoint and parse the response into a
+// CloudBootstrapConfig struct.
+func (c *hcp) fetchAgentBootstrapConfig(t *testing.T) string {
+	cfg, err := c.HCPConfig()
+	require.NoError(t, err)
+	logger.Log(t, "Fetching Consul cluster configuration from HCP")
+	httpClientCfg := httpclient.Config{
+		HCPConfig: cfg,
+	}
+	clientRuntime, err := httpclient.New(httpClientCfg)
+	require.NoError(t, err)
+
+	hcpgnmClient := hcpgnm.New(clientRuntime, nil)
+	clusterResource, err := resource.FromString(c.ResourceID)
+	require.NoError(t, err)
+
+	params := hcpgnm.NewAgentBootstrapConfigParams().
+		WithID(clusterResource.ID).
+		WithLocationOrganizationID(clusterResource.Organization).
+		WithLocationProjectID(clusterResource.Project)
+
+	resp, err := hcpgnmClient.AgentBootstrapConfig(params, nil)
+	require.NoError(t, err)
+
+	bootstrapConfig := resp.GetPayload()
+	logger.Log(t, "HCP configuration successfully fetched.")
+
+	return c.parseBootstrapConfigResponse(t, bootstrapConfig)
+}
+
+// ConsulConfig represents 'cluster.consul_config' in the response
+// fetched from the agent bootstrap config endpoint in HCP.
+type ConsulConfig struct {
+	ACL ACL `json:"acl"`
+}
+
+// ACL represents 'cluster.consul_config.acl' in the response
+// fetched from the agent bootstrap config endpoint in HCP.
+type ACL struct {
+	Tokens Tokens `json:"tokens"`
+}
+
+// Tokens represents 'cluster.consul_config.acl.tokens' in the
+// response fetched from the agent bootstrap config endpoint in HCP.
+type Tokens struct {
+	Agent             string `json:"agent"`
+	InitialManagement string `json:"initial_management"`
+}
+
+// parseBootstrapConfigResponse unmarshals the boostrap parseBootstrapConfigResponse
+// and also sets the HCPConfig values to return CloudBootstrapConfig struct.
+func (c *hcp) parseBootstrapConfigResponse(t *testing.T, bootstrapRepsonse *models.HashicorpCloudGlobalNetworkManager20220215AgentBootstrapResponse) string {
+
+	var consulConfig ConsulConfig
+	err := json.Unmarshal([]byte(bootstrapRepsonse.Bootstrap.ConsulConfig), &consulConfig)
+	require.NoError(t, err)
+
+	return consulConfig.ACL.Tokens.InitialManagement
+}
+
+func (c *hcp) HCPConfig(opts ...hcpcfg.HCPConfigOption) (hcpcfg.HCPConfig, error) {
+	if c.ClientID != "" && c.ClientSecret != "" {
+		opts = append(opts, hcpcfg.WithClientCredentials(c.ClientID, c.ClientSecret))
+	}
+	if c.AuthURL != "" {
+		opts = append(opts, hcpcfg.WithAuth(c.AuthURL, &tls.Config{}))
+	}
+	if c.APIHostname != "" {
+		opts = append(opts, hcpcfg.WithAPI(c.APIHostname, &tls.Config{}))
+	}
+	if c.ScadaAddress != "" {
+		opts = append(opts, hcpcfg.WithSCADA(c.ScadaAddress, &tls.Config{}))
+	}
+	opts = append(opts, hcpcfg.FromEnv(), hcpcfg.WithoutBrowserLogin())
+	return hcpcfg.NewHCPConfig(opts...)
+}

acceptance/tests/fixtures/bases/cloud/service-intentions/acl.yaml

@@ -0,0 +1,15 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ServiceIntentions
+metadata:
+  name: consul-telemetry-collector
+spec:
+  destination:
+    name: 'consul-telemetry-collector'
+  sources:
+    - name: '*'
+      action: allow
+
+
+

acceptance/tests/fixtures/bases/cloud/service-intentions/kustomization.yaml

@@ -0,0 +1,5 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - acl.yaml

charts/consul/templates/server-statefulset.yaml

@@ -241,6 +241,7 @@ spec:
       containers:
         - name: consul
           image: "{{ default .Values.global.image .Values.server.image }}"
+          imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           env:
             - name: ADVERTISE_IP
               valueFrom:

charts/consul/templates/telemetry-collector-deployment.yaml

@@ -150,7 +150,7 @@ spec:
       containers:
       - name: consul-telemetry-collector
         image: {{ .Values.telemetryCollector.image }}
-        imagePullPolicy: Always
+        imagePullPolicy: {{ .Values.global.imagePullPolicy }}
         ports:
         - containerPort: 9090
           name: metrics