feat: split agent & controller image && reduce image size

Signed-off-by: bingshen.wbs <[email protected]>
BSWANG · Feb 27, 2024 · c78ba8f · c78ba8f
1 parent 0ad34dc
commit c78ba8f
Show file tree

Hide file tree

Showing 10 changed files with 86 additions and 59 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -29,18 +29,21 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Cache Docker layers
-        uses: actions/cache@v2
+      - name: Docker meta agent
+        id: meta-agent
+        uses: docker/metadata-action@v3
         with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-      - name: Docker meta
-        id: meta
+          images: ${{ env.REGISTRY }}/${{ env.OWNER }}/agent
+          tags: |
+            type=raw,value={{date 'YYYYMMDD'}}-{{sha}}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{raw}}
+            type=raw,value=latest,enable={{is_default_branch}}
+      - name: Docker meta controller
+        id: meta-controller
         uses: docker/metadata-action@v3
         with:
-          images: ${{ env.REGISTRY }}/${{ env.OWNER }}/kubeskoop
+          images: ${{ env.REGISTRY }}/${{ env.OWNER }}/controller
           tags: |
             type=raw,value={{date 'YYYYMMDD'}}-{{sha}}
             type=semver,pattern={{version}}
@@ -54,12 +57,39 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
         if: ${{ github.event_name != 'pull_request' && github.event.action != 'unassigned' }}
 
-      - name: Build and push
+      - name: Go Build Cache for Docker
+        uses: actions/cache@v3
+        with:
+          path: go-build-cache
+          key: ${{ runner.os }}-go-build-cache-${{ hashFiles('**/go.sum') }}
+      - name: inject go-build-cache into docker
+        # v1 was composed of two actions: "inject" and "extract".
+        # v2 is unified to a single action.
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: go-build-cache
+
+      - name: Build and push agent
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta-agent.outputs.tags }}
+          target: agent
+          labels: ${{ steps.meta-agent.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Build and push controller
         uses: docker/build-push-action@v2
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-controller.outputs.tags }}
+          target: controller
+          labels: ${{ steps.meta-controller.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/Dockerfile b/Dockerfile
@@ -11,15 +11,19 @@ RUN if [ ! -z "$ALPINE_MIRROR" ]; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.
     go env -w GOPROXY=$GOPROXY
 
 WORKDIR /go/src/github.com/alibaba/kubeskoop/
+RUN go env -w GOMODCACHE=/root/.cache/go-build
+COPY go.mod go.sum /go/src/github.com/alibaba/kubeskoop/
+RUN --mount=type=cache,target=/root/.cache/go-build go mod download
+
 ADD . /go/src/github.com/alibaba/kubeskoop/
-RUN mkdir -p bin && make all
+RUN --mount=type=cache,target=/root/.cache/go-build mkdir -p bin && make all
 
 FROM --platform=linux/amd64 docker.io/library/node:20.9.0-alpine as build-ui
 WORKDIR /webconsole
 ADD ./webui /webconsole
 RUN yarn install && yarn build
 
-FROM docker.io/library/alpine:3.19
+FROM docker.io/library/alpine:3.19 as base
 ARG GOPROXY
 ARG ALPINE_MIRROR
 ENV GOPROXY=$GOPROXY
@@ -35,15 +39,18 @@ RUN if [ ! -z "$ALPINE_MIRROR" ]; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.
     strace \
     tcpdump \
     curl \
-    bash
+    bash && \
+    rm -rf /var/cache/apk/*
 
+FROM base as agent
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/inspector /bin/inspector
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/pod-collector /bin/pod-collector
+COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/btfhack /bin/btfhack
+
+FROM base as controller
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/controller /bin/controller
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/skoop /bin/skoop
-COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/btfhack /bin/btfhack
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/webconsole /bin/webconsole
-
 COPY --from=build-ui /webconsole/build /var/www
 
 COPY tools/scripts/* /bin/

diff --git a/Makefile b/Makefile
@@ -1,5 +1,3 @@
-SKOOP_REPO ?= kubeskoop/kubeskoop
-
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # This is a requirement for 'setup-envtest.sh' in the test target.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
@@ -11,7 +9,7 @@ TARGETARCH?=amd64
 
 TAG?=${shell git describe --tags --abbrev=7}
 GIT_COMMIT=${shell git rev-parse HEAD}
-ldflags="-X $(VERSION_PKG).Version=$(TAG) -X $(VERSION_PKG).Commit=${GIT_COMMIT}"
+ldflags="-X $(VERSION_PKG).Version=$(TAG) -X $(VERSION_PKG).Commit=${GIT_COMMIT} -w -s"
 
 .PHONY: all
 all: build-exporter build-skoop build-controller build-collector build-btfhack build-webconsole
@@ -38,7 +36,7 @@ build-collector:
 
 .PHONY: build-controller
 build-controller:
-	go build -o bin/controller -ldflags $(ldflags) ./cmd/controller
+	CGO_ENABLED=0 go build -o bin/controller -ldflags $(ldflags) ./cmd/controller
 
 .PHONY: build-btfhack
 build-btfhack:
@@ -51,11 +49,3 @@ build-webconsole:
 .PHONY: generate-btf
 generate-btf:
 	go generate ./pkg/exporter/probe/...
-
-.PHONY: image
-image: ## build kubeskoop image
-	docker build -t $(SKOOP_REPO):$(TAG) .
-
-.PHONY: push
-push: image ## push kubeskoop image
-	docker push $(SKOOP_REPO):$(TAG)
diff --git a/deploy/helm/README.md b/deploy/helm/README.md
@@ -38,20 +38,20 @@ curl {{kubeskoop-exporter的pod ip}}:9102/status |jq .
 
 ## VARIABLES
 
-| Setting                      | Description                                                                                                        | Default                                         |
-|------------------------------|--------------------------------------------------------------------------------------------------------------------| ----------------------------------------------- |
-| name                         | DaemonSet name of KubeSkoop exporter.                                                                              | `kubeskoop-exporter`                            |
-| debugmode                    | Enable `debugmode` for kubeskoop-exporter, with debug interface, debug log level and pprof support.                | `false`                                         |
-| appName                      | Pod `app` label.                                                                                                   | `kubeskoop-exporter`                            |
+| Setting                      | Description                                                                                                        | Default                                                   |
+|------------------------------|--------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------|
+| name                         | DaemonSet name of KubeSkoop exporter.                                                                              | `kubeskoop-exporter`                                      |
+| debugmode                    | Enable `debugmode` for kubeskoop-exporter, with debug interface, debug log level and pprof support.                | `false`                                                   |
+| appName                      | Pod `app` label.                                                                                                   | `kubeskoop-exporter`                                      |
 | runtimeEndpoint              | CRI runtime endpoint socket, you can use  `crictl info                                                             | awk -F":" '/containerdEndpoint/ {print $2'` to obtain it. | `/run/containerd/containerd.sock`               |
-| image.repository             | Image repository for KubeSkoop exporter container.                                                                 | `kubeskoop/kubeskoop`                           |
-| image.tag                    | Image tag for KubeSkoop exporter container.                                                                        | `latest`                                        |
-| image.imagePullPolicy        | `imagePullPolicy` for KubeSkoop exporter container.                                                                | `Always`                                        |
-| initContainer.enabled        | Enable `btfhack` as initContainer to automate discover btf file when kernel does not carry btf information itself. | `true`                                          |
-| initContainer.repository     | Image repository for `btfhack` container.                                                                          | `registry.cn-hangzhou.aliyuncs.com/acs/btfhack` |
-| initContainer.tag            | Image tag for `btfhack` container.                                                                                 | `latest`                                        |
-| initContainer.imagePullPolicy | `imagePullPolicy` for `btfhack` container.                                                                         | `Always`                                        |
-| config.serverPort            | kubeskoop metrics server port, provide HTTP service.                                                               | 9102                                            |
-| config.metricsProbes         | Metric probes to enable.                                                                                           | Refer to the probe guide.                       |
-| config.eventProbes           | Event probes to enable.                                                                                            | Refer to the probe guide.                       |
-| config.eventSinks            | Sink config for events, stderr/file/loki are supported now.                                                        | 15                                              |
+| image.repository             | Image repository for KubeSkoop exporter container.                                                                 | `kubeskoop/agent`                                         |
+| image.tag                    | Image tag for KubeSkoop exporter container.                                                                        | `latest`                                                  |
+| image.imagePullPolicy        | `imagePullPolicy` for KubeSkoop exporter container.                                                                | `Always`                                                  |
+| initContainer.enabled        | Enable `btfhack` as initContainer to automate discover btf file when kernel does not carry btf information itself. | `true`                                                    |
+| initContainer.repository     | Image repository for `btfhack` container.                                                                          | `registry.cn-hangzhou.aliyuncs.com/acs/btfhack`           |
+| initContainer.tag            | Image tag for `btfhack` container.                                                                                 | `latest`                                                  |
+| initContainer.imagePullPolicy | `imagePullPolicy` for `btfhack` container.                                                                         | `Always`                                                  |
+| config.serverPort            | kubeskoop metrics server port, provide HTTP service.                                                               | 9102                                                      |
+| config.metricsProbes         | Metric probes to enable.                                                                                           | Refer to the probe guide.                                 |
+| config.eventProbes           | Event probes to enable.                                                                                            | Refer to the probe guide.                                 |
+| config.eventSinks            | Sink config for events, stderr/file/loki are supported now.                                                        | 15                                                        |
diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml
@@ -4,15 +4,15 @@ runtimeEndpoint: /run/containerd/containerd.sock
 appName: kubeskoop-exporter
 
 image:
-  repository: kubeskoop/kubeskoop
+  repository: kubeskoop/agent
   # Overrides the image tag whose default is the chart appVersion.
-  tag: v0.1.1
+  tag: v1.0.0
   imagePullPolicy: IfNotPresent
 
 initContainer:
   enabled: true
-  repository: kubeskoop/btfhack
-  tag: v0.1.1
+  repository: kubeskoop/agent
+  tag: v1.0.0
   imagePullPolicy: IfNotPresent
 
 resources:

diff --git a/deploy/resource/kubeskoop-exporter-dashboard.json b/deploy/resource/kubeskoop-exporter-dashboard.json
@@ -2046,7 +2046,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
               },
               {
                 "color": "red",

diff --git a/deploy/skoopbundle.yaml b/deploy/skoopbundle.yaml
@@ -31,15 +31,15 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       initContainers:
         - name: inspector-prepare
-          image: kubeskoop/kubeskoop:v1.0.0
+          image: kubeskoop/agent:v1.0.0
           volumeMounts:
             - name: btf-rawdata
               mountPath: /etc/net-exporter/btf
             - mountPath: /boot/
               name: boot
           command: [btfhack, discover, -p, /etc/net-exporter/btf/]
       containers:
-      - image: kubeskoop/kubeskoop:v1.0.0
+      - image: kubeskoop/agent:v1.0.0
         name: inspector
         env:
         - name: INSPECTOR_NODENAME
@@ -385,7 +385,7 @@ spec:
             name: grafana-datasources
             readOnly: false
       - name: register-dashboard
-        image: kubeskoop/kubeskoop:v1.0.0
+        image: kubeskoop/controller:v1.0.0
         env:
           - name: GRAFANA_PASSWORD
             value: "kubeskoop"
@@ -602,7 +602,7 @@ spec:
     spec:
       containers:
       - name: controller
-        image: kubeskoop/kubeskoop:v1.0.0
+        image: kubeskoop/controller:v1.0.0
         command: ["/bin/controller"]
         env:
           - name: LOKI_ENDPOINT
@@ -653,7 +653,7 @@ spec:
     spec:
       containers:
       - name: controller
-        image: kubeskoop/kubeskoop:v1.0.0
+        image: kubeskoop/controller:v1.0.0
         command: ["/bin/webconsole"]
         env:
           - name: CONTROLLER_ENDPOINT

diff --git a/deploy/vagrant-exporter/deploy/skoop-deploy.yaml b/deploy/vagrant-exporter/deploy/skoop-deploy.yaml
@@ -30,7 +30,7 @@ spec:
       hostIPC: true
       dnsPolicy: ClusterFirstWithHostNet
       containers:
-      - image: "kubeskoop/kubeskoop:latest"
+      - image: "kubeskoop/agent:latest"
         name: inspector
         env:
         - name: INSPECTOR_NODENAME
@@ -354,7 +354,7 @@ spec:
             name: grafana-datasources
             readOnly: false
       - name: register-dashboard
-        image: kubeskoop/kubeskoop:latest
+        image: kubeskoop/controller:latest
         env:
           - name: GRAFANA_PASSWORD
             value: "kubeskoop"

diff --git a/pkg/skoop/collector/manager/config.go b/pkg/skoop/collector/manager/config.go
@@ -28,7 +28,7 @@ type SimplePodCollectorConfig struct {
 }
 
 func (cc *SimplePodCollectorConfig) BindFlags(fs *pflag.FlagSet) {
-	fs.StringVarP(&cc.Image, "collector-image", "", "kubeskoop/kubeskoop:v0.1.0", "Image used for collector.")
+	fs.StringVarP(&cc.Image, "collector-image", "", "kubeskoop/agent:v1.0.0", "Image used for collector.")
 	fs.StringVarP(&cc.CollectorNamespace, "collector-namespace", "", "skoop", "Namespace where collector pods in.")
 	fs.StringVarP(&cc.RuntimeAPIAddress, "collector-cri-address", "", "", "Runtime CRI API endpoint address.")
 	fs.DurationVarP(&cc.WaitInterval, "collector-pod-wait-interval", "", 2*time.Second, "Collector pod running check interval.")

diff --git a/test/skoop/e2e/e2e_test.go b/test/skoop/e2e/e2e_test.go
@@ -28,7 +28,7 @@ func (o *options) BindFlags() {
 	flag.StringVar(&o.CloudProvider, "cloud-provider", "generic", "cloud provider of cluster")
 	flag.StringVar(&o.SkoopPath, "executable", "kubeskoop", "kubeskoop executable file")
 	flag.StringVar(&o.Testcases, "testcases", "generic", "testcases for e2e test, separated by comma.")
-	flag.StringVar(&o.CollectorImage, "collector-image", "kubeskoop/kubeskoop:v0.1.0", "collector image for skoop cli")
+	flag.StringVar(&o.CollectorImage, "collector-image", "kubeskoop/agent:v1.0.0", "collector image for skoop cli")
 	flag.StringVar(&o.ExtraDiagnoseArgs, "extra-diagnose-args", "", "extra args for skoop")
 }