feat: allow dlv debugging for lvm operator and vgmanager

Signed-off-by: Jakob Möller <[email protected]>

Makefile

@@ -182,20 +182,23 @@ ARCH ?= amd64
 all: build
 
 build: generate fmt vet ## Build manager binary.
-	GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/lvms cmd/main.go
+	GOOS=$(OS) GOARCH=$(ARCH) go build -gcflags='all=-N -l' -o bin/lvms cmd/main.go
 
 build-prometheus-alert-rules: jsonnet monitoring/mixin.libsonnet monitoring/alerts/alerts.jsonnet monitoring/alerts/*.libsonnet
 	$(JSONNET) -S monitoring/alerts/alerts.jsonnet > config/prometheus/prometheus_rules.yaml
 
 docker-build: ## Build docker image with the manager.
 	$(IMAGE_BUILD_CMD) build --platform=${OS}/${ARCH} -t ${IMG} .
 
+docker-build-debug: ## Build remote-debugging enabled docker image with the manager.
+	$(IMAGE_BUILD_CMD) build -f debug.Dockerfile --platform=${OS}/${ARCH} -t ${IMG} .
+
 docker-push: ## Push docker image with the manager.
 	$(IMAGE_BUILD_CMD) push ${IMG}
 
 lvms-must-gather:
 	@echo "Building the lvms-must-gather image"
-	$(IMAGE_BUILD_CMD) build -f must-gather/Dockerfile -t "${MUST_GATHER_IMG}" must-gather/
+	$(IMAGE_BUILD_CMD) build  -f must-gather/Dockerfile -t "${MUST_GATHER_IMG}" must-gather/
 
 ##@ Deployment
 
@@ -210,6 +213,11 @@ deploy: update-mgr-env manifests kustomize ## Deploy controller to the K8s clust
 	cd config/webhook && $(KUSTOMIZE) edit set nameprefix ${MANAGER_NAME_PREFIX}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
+deploy-debug: update-mgr-env manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} && $(KUSTOMIZE) edit set nameprefix ${MANAGER_NAME_PREFIX}
+	cd config/webhook && $(KUSTOMIZE) edit set nameprefix ${MANAGER_NAME_PREFIX}
+	$(KUSTOMIZE) build config/debug | kubectl apply -f -
+
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/default | kubectl delete -f -
 

cmd/operator/operator.go

@@ -55,13 +55,17 @@ const (
 	DefaultEnableLeaderElection = false
 )
 
+var DefaultVGManagerCommand = []string{"/lvms", "vgmanager"}
+
 type Options struct {
 	Scheme   *runtime.Scheme
 	SetupLog logr.Logger
 
 	diagnosticsAddr      string
 	healthProbeAddr      string
 	enableLeaderElection bool
+
+	vgManagerCommand []string
 }
 
 // NewCmd creates a new CLI command
@@ -88,6 +92,10 @@ func NewCmd(opts *Options) *cobra.Command {
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.",
 	)
 
+	cmd.Flags().StringSliceVar(
+		&opts.vgManagerCommand, "vgmanager-cmd", DefaultVGManagerCommand, "The address the probe endpoint binds to.",
+	)
+
 	return cmd
 }
 
@@ -165,6 +173,7 @@ func run(cmd *cobra.Command, _ []string, opts *Options) error {
 		Namespace:                        operatorNamespace,
 		TopoLVMLeaderElectionPassthrough: leaderElectionConfig,
 		EnableSnapshotting:               enableSnapshotting,
+		VGManagerCommand:                 opts.vgManagerCommand,
 	}).SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("unable to create LVMCluster controller: %w", err)
 	}

config/debug/kustomization.yaml

@@ -0,0 +1,6 @@
+patchesStrategicMerge:
+  - manager_debug_patch.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../default

config/debug/manager_debug_patch.yaml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: operator
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+        - name: manager
+          command:
+            - "/usr/sbin/dlv"
+            - "exec"
+            - "--listen=:2345"
+            - "--headless=true"
+            - "--log=true"
+            - "--accept-multiclient"
+            - "--api-version=2"
+            - "--continue"
+            - "/usr/sbin/lvms"
+            - "--"
+            - "operator"
+          args:
+            - "--vgmanager-cmd=/usr/sbin/dlv,exec,--listen=:2345,--headless=true,--log=true,--accept-multiclient,--api-version=2,--continue,/usr/sbin/lvms,--,vgmanager"
+          ports:
+            - containerPort: 2345
+              name: "debug"

config/manager/kustomization.yaml

@@ -7,6 +7,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/lvms_dev/lvms-operator
+  newName: quay.io/jmoller/lvms-operator
   newTag: latest
 namePrefix: lvms-

debug.Dockerfile

@@ -0,0 +1,52 @@
+# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETPLATFORM
+FROM golang:1.20 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# since we use vendoring we don't need to redownload our dependencies every time. Instead we can simply
+# reuse our vendored directory and verify everything is good. If not we can abort here and ask for a revendor.
+COPY vendor vendor/
+RUN go mod verify
+
+# Copy the go source
+COPY api/ api/
+COPY controllers/ controllers/
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+
+ENV GOARCH=$TARGETARCH
+ENV GOOS=$TARGETOS
+ENV CGO_ENABLED=0
+
+# Build
+RUN go build -gcflags "all=-N -l" -mod=vendor -a -o lvms cmd/main.go
+
+FROM golang:1.20 as dlv
+RUN go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest
+
+# vgmanager needs 'nsenter' and other basic linux utils to correctly function
+FROM --platform=$TARGETPLATFORM registry.access.redhat.com/ubi9/ubi-minimal:9.2
+
+# Update the image to get the latest CVE updates
+RUN microdnf update -y && \
+    microdnf install -y util-linux && \
+    microdnf clean all
+
+
+WORKDIR /app
+
+COPY --from=builder /workspace/lvms /usr/sbin/lvms
+COPY --from=dlv /go/bin/dlv /usr/sbin/dlv
+
+USER 65532:65532
+
+EXPOSE 2345
+
+# '/usr/sbin/dlv' is the entrypoint for all LVMS binaries
+ENTRYPOINT ["/usr/sbin/dlv"]

internal/controllers/lvmcluster/lvmcluster_controller.go

@@ -64,6 +64,9 @@ type LVMClusterReconciler struct {
 	Namespace          string
 	ImageName          string
 
+	// VGManagerCommand is the command that will be used to start vgmanager
+	VGManagerCommand []string
+
 	// TopoLVMLeaderElectionPassthrough uses the given leaderElection when initializing TopoLVM to synchronize
 	// leader election configuration
 	TopoLVMLeaderElectionPassthrough configv1.LeaderElection
@@ -85,6 +88,10 @@ func (r *LVMClusterReconciler) SnapshotsEnabled() bool {
 	return r.EnableSnapshotting
 }
 
+func (r *LVMClusterReconciler) GetVGManagerCommand() []string {
+	return r.VGManagerCommand
+}
+
 func (r *LVMClusterReconciler) GetTopoLVMLeaderElectionPassthrough() configv1.LeaderElection {
 	return r.TopoLVMLeaderElectionPassthrough
 }

internal/controllers/lvmcluster/resource/manager.go

@@ -14,6 +14,7 @@ type Reconciler interface {
 	GetNamespace() string
 	GetImageName() string
 	SnapshotsEnabled() bool
+	GetVGManagerCommand() []string
 
 	// GetTopoLVMLeaderElectionPassthrough uses the given leaderElection when initializing TopoLVM to synchronize
 	// leader election configuration

internal/controllers/lvmcluster/resource/vgmanager.go

@@ -47,7 +47,7 @@ func (v vgManager) EnsureCreated(r Reconciler, ctx context.Context, lvmCluster *
 	logger := log.FromContext(ctx).WithValues("resourceManager", v.GetName())
 
 	// get desired daemonset spec
-	dsTemplate := newVGManagerDaemonset(lvmCluster, r.GetNamespace(), r.GetImageName())
+	dsTemplate := newVGManagerDaemonset(lvmCluster, r.GetNamespace(), r.GetImageName(), r.GetVGManagerCommand())
 
 	// create desired daemonset or update mutable fields on existing one
 	ds := &appsv1.DaemonSet{

internal/controllers/lvmcluster/resource/vgmanager_daemonset.go

@@ -137,16 +137,16 @@ var (
 )
 
 // newVGManagerDaemonset returns the desired vgmanager daemonset for a given LVMCluster
-func newVGManagerDaemonset(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, vgImage string) appsv1.DaemonSet {
+func newVGManagerDaemonset(lvmCluster *lvmv1alpha1.LVMCluster, namespace, vgImage string, command []string) appsv1.DaemonSet {
 	// aggregate nodeSelector and tolerations from all deviceClasses
 	nodeSelector, tolerations := selector.ExtractNodeSelectorAndTolerations(lvmCluster)
 	volumes := []corev1.Volume{LVMDConfVol, DevHostDirVol, UDevHostDirVol, SysHostDirVol, MetricsCertsDirVol}
 	volumeMounts := []corev1.VolumeMount{LVMDConfVolMount, DevHostDirVolMount, UDevHostDirVolMount, SysHostDirVolMount, MetricsCertsDirVolMount}
 	privileged := true
 	var zero int64 = 0
 
-	command := []string{
-		"/lvms", "vgmanager",
+	if len(command) == 0 {
+		command = []string{"/lvms", "vgmanager"}
 	}
 
 	resourceRequirements := corev1.ResourceRequirements{