feat: split agent & controller image && reduce image size

Signed-off-by: bingshen.wbs <[email protected]>

.github/workflows/build.yml

@@ -29,18 +29,21 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Cache Docker layers
-        uses: actions/cache@v2
+      - name: Docker meta agent
+        id: meta-agent
+        uses: docker/metadata-action@v3
         with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-      - name: Docker meta
-        id: meta
+          images: ${{ env.REGISTRY }}/${{ env.OWNER }}/agent
+          tags: |
+            type=raw,value={{date 'YYYYMMDD'}}-{{sha}}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{raw}}
+            type=raw,value=latest,enable={{is_default_branch}}
+      - name: Docker meta controller
+        id: meta-controller
         uses: docker/metadata-action@v3
         with:
-          images: ${{ env.REGISTRY }}/${{ env.OWNER }}/kubeskoop
+          images: ${{ env.REGISTRY }}/${{ env.OWNER }}/controller
           tags: |
             type=raw,value={{date 'YYYYMMDD'}}-{{sha}}
             type=semver,pattern={{version}}
@@ -54,12 +57,39 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
         if: ${{ github.event_name != 'pull_request' && github.event.action != 'unassigned' }}
 
-      - name: Build and push
+      - name: Go Build Cache for Docker
+        uses: actions/cache@v3
+        with:
+          path: go-build-cache
+          key: ${{ runner.os }}-go-build-cache-${{ hashFiles('**/go.sum') }}
+      - name: inject go-build-cache into docker
+        # v1 was composed of two actions: "inject" and "extract".
+        # v2 is unified to a single action.
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: go-build-cache
+
+      - name: Build and push agent
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta-agent.outputs.tags }}
+          target: agent
+          labels: ${{ steps.meta-agent.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Build and push controller
         uses: docker/build-push-action@v2
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-controller.outputs.tags }}
+          target: controller
+          labels: ${{ steps.meta-controller.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Dockerfile

@@ -11,15 +11,19 @@ RUN if [ ! -z "$ALPINE_MIRROR" ]; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.
     go env -w GOPROXY=$GOPROXY
 
 WORKDIR /go/src/github.com/alibaba/kubeskoop/
+RUN go env -w GOMODCACHE=/root/.cache/go-build
+COPY go.mod go.sum /go/src/github.com/alibaba/kubeskoop/
+RUN --mount=type=cache,target=/root/.cache/go-build go mod download
+
 ADD . /go/src/github.com/alibaba/kubeskoop/
-RUN mkdir -p bin && make all
+RUN --mount=type=cache,target=/root/.cache/go-build mkdir -p bin && make all
 
 FROM --platform=linux/amd64 docker.io/library/node:20.9.0-alpine as build-ui
 WORKDIR /webconsole
 ADD ./webui /webconsole
 RUN yarn install && yarn build
 
-FROM docker.io/library/alpine:3.19
+FROM docker.io/library/alpine:3.19 as base
 ARG GOPROXY
 ARG ALPINE_MIRROR
 ENV GOPROXY=$GOPROXY
@@ -35,15 +39,18 @@ RUN if [ ! -z "$ALPINE_MIRROR" ]; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.
     strace \
     tcpdump \
     curl \
-    bash
+    bash && \
+    rm -rf /var/cache/apk/*
 
+FROM base as agent
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/inspector /bin/inspector
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/pod-collector /bin/pod-collector
+COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/btfhack /bin/btfhack
+
+FROM base as controller
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/controller /bin/controller
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/skoop /bin/skoop
-COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/btfhack /bin/btfhack
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/webconsole /bin/webconsole
-
 COPY --from=build-ui /webconsole/build /var/www
 
 COPY tools/scripts/* /bin/

Makefile

@@ -11,7 +11,7 @@ TARGETARCH?=amd64
 
 TAG?=${shell git describe --tags --abbrev=7}
 GIT_COMMIT=${shell git rev-parse HEAD}
-ldflags="-X $(VERSION_PKG).Version=$(TAG) -X $(VERSION_PKG).Commit=${GIT_COMMIT}"
+ldflags="-X $(VERSION_PKG).Version=$(TAG) -X $(VERSION_PKG).Commit=${GIT_COMMIT} -w -s"
 
 .PHONY: all
 all: build-exporter build-skoop build-controller build-collector build-btfhack build-webconsole

deploy/resource/kubeskoop-exporter-dashboard.json

@@ -2046,7 +2046,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
               },
               {
                 "color": "red",

deploy/skoopbundle.yaml

@@ -31,15 +31,15 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       initContainers:
         - name: inspector-prepare
-          image: kubeskoop/kubeskoop:v1.0.0
+          image: kubeskoop/agent:v1.0.0
           volumeMounts:
             - name: btf-rawdata
               mountPath: /etc/net-exporter/btf
             - mountPath: /boot/
               name: boot
           command: [btfhack, discover, -p, /etc/net-exporter/btf/]
       containers:
-      - image: kubeskoop/kubeskoop:v1.0.0
+      - image: kubeskoop/agent:v1.0.0
         name: inspector
         env:
         - name: INSPECTOR_NODENAME
@@ -385,7 +385,7 @@ spec:
             name: grafana-datasources
             readOnly: false
       - name: register-dashboard
-        image: kubeskoop/kubeskoop:v1.0.0
+        image: kubeskoop/controller:v1.0.0
         env:
           - name: GRAFANA_PASSWORD
             value: "kubeskoop"
@@ -602,7 +602,7 @@ spec:
     spec:
       containers:
       - name: controller
-        image: kubeskoop/kubeskoop:v1.0.0
+        image: kubeskoop/controller:v1.0.0
         command: ["/bin/controller"]
         env:
           - name: LOKI_ENDPOINT
@@ -653,7 +653,7 @@ spec:
     spec:
       containers:
       - name: controller
-        image: kubeskoop/kubeskoop:v1.0.0
+        image: kubeskoop/controller:v1.0.0
         command: ["/bin/webconsole"]
         env:
           - name: CONTROLLER_ENDPOINT

deploy/vagrant-exporter/deploy/skoop-deploy.yaml

@@ -30,7 +30,7 @@ spec:
       hostIPC: true
       dnsPolicy: ClusterFirstWithHostNet
       containers:
-      - image: "kubeskoop/kubeskoop:latest"
+      - image: "kubeskoop/agent:latest"
         name: inspector
         env:
         - name: INSPECTOR_NODENAME
@@ -354,7 +354,7 @@ spec:
             name: grafana-datasources
             readOnly: false
       - name: register-dashboard
-        image: kubeskoop/kubeskoop:latest
+        image: kubeskoop/controller:latest
         env:
           - name: GRAFANA_PASSWORD
             value: "kubeskoop"

pkg/skoop/collector/manager/config.go

@@ -28,7 +28,7 @@ type SimplePodCollectorConfig struct {
 }
 
 func (cc *SimplePodCollectorConfig) BindFlags(fs *pflag.FlagSet) {
-	fs.StringVarP(&cc.Image, "collector-image", "", "kubeskoop/kubeskoop:v0.1.0", "Image used for collector.")
+	fs.StringVarP(&cc.Image, "collector-image", "", "kubeskoop/agent:v1.0.0", "Image used for collector.")
 	fs.StringVarP(&cc.CollectorNamespace, "collector-namespace", "", "skoop", "Namespace where collector pods in.")
 	fs.StringVarP(&cc.RuntimeAPIAddress, "collector-cri-address", "", "", "Runtime CRI API endpoint address.")
 	fs.DurationVarP(&cc.WaitInterval, "collector-pod-wait-interval", "", 2*time.Second, "Collector pod running check interval.")

test/skoop/e2e/e2e_test.go

@@ -28,7 +28,7 @@ func (o *options) BindFlags() {
 	flag.StringVar(&o.CloudProvider, "cloud-provider", "generic", "cloud provider of cluster")
 	flag.StringVar(&o.SkoopPath, "executable", "kubeskoop", "kubeskoop executable file")
 	flag.StringVar(&o.Testcases, "testcases", "generic", "testcases for e2e test, separated by comma.")
-	flag.StringVar(&o.CollectorImage, "collector-image", "kubeskoop/kubeskoop:v0.1.0", "collector image for skoop cli")
+	flag.StringVar(&o.CollectorImage, "collector-image", "kubeskoop/agent:v1.0.0", "collector image for skoop cli")
 	flag.StringVar(&o.ExtraDiagnoseArgs, "extra-diagnose-args", "", "extra args for skoop")
 }