feat: metric plugin system based on hashicorp go-plugin

.gitignore

@@ -13,6 +13,7 @@ junit.xml
 rerunreport.txt
 site/
 vendor/
+plugin-bin/
 # static
 server/static/*
 !server/static/.gitkeep

Dockerfile

@@ -84,7 +84,7 @@ CMD ["dashboard"]
 ####################################################################################################
 # Final image
 ####################################################################################################
-FROM scratch
+FROM gcr.io/distroless/static-debian11
 
 COPY --from=argo-rollouts-build /go/src/github.com/argoproj/argo-rollouts/dist/rollouts-controller /bin/
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/

Dockerfile.dev

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-FROM scratch    
+FROM gcr.io/distroless/static-debian11
 
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY rollouts-controller-linux-amd64 /bin/rollouts-controller

Makefile

@@ -209,7 +209,7 @@ lint: go-mod-vendor
 
 .PHONY: test
 test: test-kustomize
-	go test -covermode=count -coverprofile=coverage.out ${TEST_TARGET}
+	@make test-unit
 
 .PHONY: test-kustomize
 test-kustomize:
@@ -225,7 +225,7 @@ test-e2e: install-devtools-local
 
 .PHONY: test-unit
  test-unit: install-devtools-local
-	${DIST_DIR}/gotestsum --junitfile=junit.xml --format=testname --packages="./..." -- -covermode=count -coverprofile=coverage.out ./...
+	${DIST_DIR}/gotestsum --junitfile=junit.xml --format=testname -- -covermode=count -coverprofile=coverage.out `go list ./... | grep -v ./test/cmd/sample-metrics-plugin`
 
 
 .PHONY: coverage
@@ -279,3 +279,10 @@ trivy:
 .PHONY: checksums
 checksums:
 	shasum -a 256 ./dist/kubectl-argo-rollouts-* | awk -F './dist/' '{print $$1 $$2}' > ./dist/argo-rollouts-checksums.txt
+
+# Build sample plugin with debug info
+# https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html
+.PHONY: build-sample-metric-plugin-debug
+build-sample-metric-plugin-debug:
+	go build -gcflags="all=-N -l" -o metric-plugin test/cmd/sample-metrics-plugin/main.go
+

analysis/analysis.go

@@ -327,7 +327,7 @@ func (c *Controller) runMeasurements(run *v1alpha1.AnalysisRun, tasks []metricTa
 
 			provider, err := c.newProvider(*logger, t.metric)
 			if err != nil {
-				log.Errorf("Error in getting provider :%v", err)
+				log.Errorf("Error in getting metric provider :%v", err)
 				return err
 			}
 			if metricResult == nil {

analysis/controller.go

@@ -5,6 +5,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/argoproj/argo-rollouts/metric"
+
 	unstructuredutil "github.com/argoproj/argo-rollouts/utils/unstructured"
 
 	log "github.com/sirupsen/logrus"
@@ -44,7 +46,7 @@ type Controller struct {
 
 	metricsServer *metrics.MetricsServer
 
-	newProvider func(logCtx log.Entry, metric v1alpha1.Metric) (metricproviders.Provider, error)
+	newProvider func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error)
 
 	// used for unit testing
 	enqueueAnalysis      func(obj interface{})

analysis/controller_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/argoproj/argo-rollouts/metric"
+
 	timeutil "github.com/argoproj/argo-rollouts/utils/time"
 
 	"github.com/argoproj/argo-rollouts/utils/queue"
@@ -25,7 +27,6 @@ import (
 	"k8s.io/client-go/util/workqueue"
 
 	"github.com/argoproj/argo-rollouts/controller/metrics"
-	"github.com/argoproj/argo-rollouts/metricproviders"
 	"github.com/argoproj/argo-rollouts/metricproviders/mocks"
 	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	"github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake"
@@ -129,7 +130,7 @@ func (f *fixture) newController(resync resyncFunc) (*Controller, informers.Share
 		c.enqueueAnalysis(obj)
 	}
 	f.provider = &mocks.Provider{}
-	c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metricproviders.Provider, error) {
+	c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) {
 		return f.provider, nil
 	}
 

cmd/rollouts-controller/main.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/argoproj/pkg/kubeclientmetrics"
 	smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -20,8 +21,6 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
 	"k8s.io/client-go/tools/clientcmd"
 
-	"github.com/argoproj/pkg/kubeclientmetrics"
-
 	"github.com/argoproj/argo-rollouts/controller"
 	"github.com/argoproj/argo-rollouts/controller/metrics"
 	jobprovider "github.com/argoproj/argo-rollouts/metricproviders/job"

controller/controller.go

@@ -9,9 +9,15 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/apimachinery/pkg/util/wait"
+	"github.com/argoproj/argo-rollouts/utils/plugin"
 
 	istioutil "github.com/argoproj/argo-rollouts/utils/istio"
+
+	rolloutsConfig "github.com/argoproj/argo-rollouts/utils/config"
+	goPlugin "github.com/hashicorp/go-plugin"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
 	"k8s.io/client-go/dynamic/dynamicinformer"
 	kubeinformers "k8s.io/client-go/informers"
 
@@ -352,6 +358,16 @@ func NewManager(
 		istioPrimaryDynamicClient:          istioPrimaryDynamicClient,
 	}
 
+	_, err := rolloutsConfig.InitializeConfig(kubeclientset, defaults.DefaultRolloutsConfigMapName)
+	if err != nil {
+		log.Fatalf("Failed to init config: %v", err)
+	}
+
+	err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{})
+	if err != nil {
+		log.Fatalf("Failed to download plugins: %v", err)
+	}
+
 	return cm
 }
 
@@ -423,6 +439,7 @@ func (c *Manager) Run(ctx context.Context, rolloutThreadiness, serviceThreadines
 		})
 	}
 	log.Info("Shutting down workers")
+	goPlugin.CleanupClients()
 
 	c.serviceWorkqueue.ShutDownWithDrain()
 	c.ingressWorkqueue.ShutDownWithDrain()

docs/analysis/plugins.md

@@ -0,0 +1,78 @@
+# Metric Plugins
+
+!!! important Available since v1.5
+
+Argo Rollouts supports getting analysis metrics via 3rd party plugin system. This allows users to extend the capabilities of Rollouts 
+to support metric providers that are not natively supported. Rollout's uses a plugin library called
+[go-plugin](https://github.com/hashicorp/go-plugin) to do this. You can find a sample plugin 
+here: [sample-rollouts-metric-plugin](https://github.com/argoproj-labs/sample-rollouts-metric-plugin)
+
+## Using a Metric Plugin
+
+There are two methods of installing and using an argo rollouts plugin. The first method is to mount up the plugin executable
+into the rollouts controller container. The second method is to use a HTTP(S) server to host the plugin executable.
+
+### Mounting the plugin executable into the rollouts controller container
+
+To use this method, you will need to build or download the plugin executable and then mount it into the rollouts controller container.
+The plugin executable must be mounted into the rollouts controller container at the path specified by the `--metric-plugin-location` flag.
+
+There are a few ways to mount the plugin executable into the rollouts controller container. Some of these will depend on your
+particular infrastructure. Here are a few methods:
+
+* Using an init container to download the plugin executable
+* Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc.
+* Building the plugin into the rollouts controller container
+
+Then you can use the configmap to point to the plugin executable. Example:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argo-rollouts-config
+data:
+  plugins: |-
+    metrics:
+    - name: "prometheus" # name the plugin uses to find this configuration, it must match the name required by the plugin
+      pluginLocation: "file://./my-custom-plugin" # supports http(s):// urls and file://
+```
+
+### Using a HTTP(S) server to host the plugin executable
+
+Argo Rollouts supports downloading the plugin executable from a HTTP(S) server. To use this method, you will need to 
+configure the controller via the `argo-rollouts-config` configmap and set `pluginLocation` to a http(s) url. Example:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argo-rollouts-config
+data:
+  plugins: |-
+    metrics:
+    - name: "prometheus" # name the plugin uses to find this configuration, it must match the name required by the plugin
+      pluginLocation: "https://github.com/argoproj-labs/sample-rollouts-metric-plugin/releases/download/v0.0.3/metric-plugin-linux-amd64" # supports http(s):// urls and file://
+      pluginSha256: "08f588b1c799a37bbe8d0fc74cc1b1492dd70b2c" #optional sha256 checksum of the plugin executable
+```
+
+## Some words of caution
+
+Depending on which method you use to install and the plugin, there are some things to be aware of.
+The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using
+a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts
+controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until 
+the server hosting the plugin is available again.
+
+Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running
+Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets
+deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. However,
+it is up to you to define your risk for and decide how you want to install the plugin executable.
+
+## List of Available Plugins (alphabetical order)
+
+#### Add Your Plugin Here
+  * If you have created a plugin, please submit a PR to add it to this list.
+#### [sample-rollouts-metric-plugin](https://github.com/argoproj-labs/sample-rollouts-metric-plugin)
+  * This is just a sample plugin that can be used as a starting point for creating your own plugin. 
+It is not meant to be used in production. It is based on the built-in prometheus provider.

docs/features/kustomize/rollout_cr_schema.json

@@ -4165,6 +4165,10 @@
                                                 ],
                                                 "type": "object"
                                             },
+                                            "plugin": {
+                                                "type": "object",
+                                                "x-kubernetes-preserve-unknown-fields": true
+                                            },
                                             "prometheus": {
                                                 "properties": {
                                                     "address": {
@@ -8433,6 +8437,10 @@
                                                 ],
                                                 "type": "object"
                                             },
+                                            "plugin": {
+                                                "type": "object",
+                                                "x-kubernetes-preserve-unknown-fields": true
+                                            },
                                             "prometheus": {
                                                 "properties": {
                                                     "address": {
@@ -12701,6 +12709,10 @@
                                                 ],
                                                 "type": "object"
                                             },
+                                            "plugin": {
+                                                "type": "object",
+                                                "x-kubernetes-preserve-unknown-fields": true
+                                            },
                                             "prometheus": {
                                                 "properties": {
                                                     "address": {

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.2
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/hashicorp/go-plugin v1.4.8
 	github.com/influxdata/influxdb-client-go/v2 v2.12.2
 	github.com/juju/ansiterm v1.0.0
 	github.com/mitchellh/mapstructure v1.5.0
@@ -92,6 +93,7 @@ require (
 	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
+	github.com/fatih/color v1.7.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
@@ -114,7 +116,9 @@ require (
 	github.com/gregdel/pushover v1.1.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v0.14.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
+	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -130,6 +134,7 @@ require (
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
@@ -139,6 +144,7 @@ require (
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
+	github.com/oklog/run v1.0.0 // indirect
 	github.com/onsi/ginkgo v1.16.4 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.13 // indirect