startTime: 2022-08-24T10:45:57Z

Unified Dockerfile Image for Spark Jobs Currently Theia supports separate dockerfiles for anomalydetector and policyrecommendation, this PR helps unify these dockerfiles resulting into lesser storage space. Unified Dockerimage now uses 1.4 GB of image space than earlier 2.4GB combined Spark Jobs constants have been moved under utils.go in order to avoid duplicacy of constants Solve #161 Signed-off-by: Tushar Tathgur <[email protected]>
antrea-io · Mar 4, 2023 · bd70023 · bd70023
1 parent 165ae97
commit bd70023
Show file tree

Hide file tree

Showing 12 changed files with 120 additions and 226 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -75,9 +75,9 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         make clickhouse-server-multi-arch
-
-  check-policy-recommendation-changes:
-    name: Check whether policy-recommendation image needs to be built based on diff
+  
+  check-spark-jobs-changes:
+    name: Check whether spark-jobs image needs to be built based on diff
     runs-on: [ubuntu-latest]
     steps:
     - uses: actions/checkout@v3
@@ -86,26 +86,26 @@ jobs:
     - uses: antrea-io/has-changes@v2
       id: check_diff
       with:
-        paths: plugins/policy-recommendation/* build/images/Dockerfile.policy-recommendation.ubuntu
+        paths: plugins/policy-recommendation/* plugins/anomaly-detection/* build/images/Dockerfile.spark-jobs.ubuntu
     outputs:
       has_changes: ${{ steps.check_diff.outputs.has_changes }}
 
-  build-policy-recommendation:
-    needs: check-policy-recommendation-changes
-    if: ${{ needs.check-policy-recommendation-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+  build-spark-jobs:
+    needs: check-spark-jobs-changes
+    if: ${{ needs.check-spark-jobs-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
     runs-on: [ubuntu-latest]
     steps:
     - uses: actions/checkout@v2
-    - name: Build policy-recommendation Docker image
-      run: make policy-recommendation
-    - name: Push policy-recommendation Docker image to registry
+    - name: Build spark-jobs Docker image
+      run: make spark-jobs
+    - name: Push spark-jobs Docker image to registry
       if: ${{ github.repository == 'antrea-io/theia' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       env:
         DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
         DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-        docker push antrea/theia-policy-recommendation:latest
+        docker push antrea/theia-spark-jobs:latest
 
   check-theia-manager-changes:
     name: Check whether theia-manager image needs to be built based on diff
@@ -137,34 +137,3 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-manager:latest
-  
-  check-anomaly-detection-changes:
-    name: Check whether anomaly-detection image needs to be built based on diff
-    runs-on: [ubuntu-latest]
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - uses: antrea-io/has-changes@v2
-      id: check_diff
-      with:
-        paths: plugins/anomaly-detection/* build/images/Dockerfile.anomaly-detection.ubuntu
-    outputs:
-      has_changes: ${{ steps.check_diff.outputs.has_changes }}
-
-  build-anomaly-detection:
-    needs: check-anomaly-detection-changes
-    if: ${{ needs.check-anomaly-detection-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
-    runs-on: [ubuntu-latest]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Build anomaly-detection Docker image
-      run: make anomaly-detection
-    - name: Push anomaly-detection Docker image to registry
-      if: ${{ github.repository == 'antrea-io/theia' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-      env:
-        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-      run: |
-        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-        docker push antrea/theia-anomaly-detection:latest
diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
@@ -50,35 +50,20 @@ jobs:
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         make clickhouse-server-multi-arch
 
-  build-policy-recommendation:
+  build-spark-jobs:
     runs-on: [ubuntu-latest]
     needs: get-version
     steps:
     - uses: actions/checkout@v2
-    - name: Build policy-recommendation Docker image and push to registry
+    - name: Build spark-jobs Docker image and push to registry
       env:
         DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
         DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
         VERSION: ${{ needs.get-version.outputs.version }}
       run: |
-        make policy-recommendation
+        make spark-jobs
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-        docker push antrea/theia-policy-recommendation:"${VERSION}"
-
-  build-anomaly-detection:
-    runs-on: [ubuntu-latest]
-    needs: get-version
-    steps:
-    - uses: actions/checkout@v2
-    - name: Build anomaly-detection Docker image and push to registry
-      env:
-        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        VERSION: ${{ needs.get-version.outputs.version }}
-      run: |
-        make anomaly-detection
-        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-        docker push antrea/theia-anomaly-detection:"${VERSION}"
+        docker push antrea/theia-saprk-jobs:"${VERSION}"
 
   build-theia-manager:
     runs-on: [ubuntu-latest]

diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -29,38 +29,21 @@ jobs:
     outputs:
       has_changes: ${{ steps.check_diff.outputs.has_changes }}
 
-  build-policy-recommendation-image:
-    name: Build Policy Recommendation image to be used for Kind e2e tests
+  build-spark-jobs-image:
+    name: Build Spark Jobs image to be used for Kind e2e tests
     needs: check-changes
     if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
     runs-on: [ ubuntu-latest ]
     steps:
     - uses: actions/checkout@v3
-    - run: make policy-recommendation
-    - name: Save Policy Recommendation image to tarball
-      run: docker save -o policy-recommendation.tar antrea/theia-policy-recommendation
-    - name: Upload Policy Recommendation image for subsequent jobs
+    - run: make spark-jobs
+    - name: Save Spark jobs image to tarball
+      run: docker save -o spark-jobs.tar antrea/theia-spark-jobs
+    - name: Upload Spark jobs image for subsequent jobs
       uses: actions/upload-artifact@v3
       with:
-        name: policy-recommendation
-        path: policy-recommendation.tar
-        retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
-
-  build-anomaly-detection-image:
-    name: Build Anomaly Detection image to be used for Kind e2e tests
-    needs: check-changes
-    if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
-    runs-on: [ ubuntu-latest ]
-    steps:
-    - uses: actions/checkout@v3
-    - run: make anomaly-detection
-    - name: Save Anomaly Detection image to tarball
-      run: docker save -o anomaly-detection.tar antrea/theia-anomaly-detection
-    - name: Upload Anomaly Detection image for subsequent jobs
-      uses: actions/upload-artifact@v3
-      with:
-        name: anomaly-detection
-        path: anomaly-detection.tar
+        name: spark-jobs
+        path: spark-jobs.tar
         retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
 
   build-clickhouse-monitor-image:
@@ -116,9 +99,8 @@ jobs:
 
   test-e2e-encap:
     name: E2e tests on a Kind cluster on Linux
-    needs: 
-      - build-policy-recommendation-image
-      - build-anomaly-detection-image
+    needs:
+      - build-spark-jobs-image
       - build-clickhouse-monitor-image
       - build-clickhouse-server-image
       - build-theia-manager-image
@@ -137,14 +119,10 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version-file: 'go.mod'
-      - name: Download Policy Recommendation images from previous jobs
+      - name: Download Spark jobs images from previous jobs
         uses: actions/download-artifact@v3
         with:
-          name: policy-recommendation
-      - name: Download Anomaly Detection images from previous jobs
-        uses: actions/download-artifact@v3
-        with:
-          name: anomaly-detection
+          name: spark-jobs
       - name: Download ClickHouse monitor images from previous jobs
         uses: actions/download-artifact@v3
         with:
@@ -159,10 +137,8 @@ jobs:
           name: theia-manager
       - name: Load Theia image
         run:  |
-          docker load -i policy-recommendation.tar
-          docker tag antrea/theia-policy-recommendation:latest projects.registry.vmware.com/antrea/theia-policy-recommendation:latest
-          docker load -i anomaly-detection.tar
-          docker tag antrea/theia-anomaly-detection:latest projects.registry.vmware.com/antrea/theia-anomaly-detection:latest
+          docker load -i spark-jobs.tar
+          docker tag antrea/theia-spark-jobs:latest projects.registry.vmware.com/antrea/theia-spark-jobs:latest
           docker load -i clickhouse-monitor.tar
           docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
           docker load -i clickhouse-server.tar
@@ -252,25 +228,19 @@ jobs:
   artifact-cleanup:
     name: Delete uploaded images
     needs:
-    - build-policy-recommendation-image
-    - build-anomaly-detection-image
+    - build-spark-jobs-image
     - build-clickhouse-monitor-image
     - build-clickhouse-server-image
     - build-theia-manager-image
     - test-e2e-encap
     - test-upgrade-from-N-1
     runs-on: [ubuntu-latest]
     steps:
-    - name: Delete policy-recommendation
-      if: ${{ needs.build-policy-recommendation-image.result == 'success' }}
-      uses: geekyeggo/delete-artifact@v1
-      with:
-        name: policy-recommendation
-    - name: Delete anomaly-detection
-      if: ${{ needs.build-anomaly-detection-image.result == 'success' }}
+    - name: Delete spark-jobs
+      if: ${{ needs.build-spark-jobs-image.result == 'success' }}
       uses: geekyeggo/delete-artifact@v1
       with:
-        name: anomaly-detection
+        name: spark-jobs
     - name: Delete clickhouse-monitor
       if: ${{ needs.build-clickhouse-monitor-image.result == 'success' }}
       uses: geekyeggo/delete-artifact@v1

diff --git a/Makefile b/Makefile
@@ -208,21 +208,15 @@ clickhouse-schema-management-plugin:
 	@mkdir -p $(BINDIR)
 	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/plugins/clickhouse-schema-management
 
-.PHONY: policy-recommendation
-policy-recommendation:
-	@echo "===> Building antrea/theia-policy-recommendation Docker image <==="
-	docker build --pull -t antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.policy-recommendation.ubuntu .
-	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) antrea/theia-policy-recommendation
-	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation
-	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION)
-
-.PHONY: anomaly-detection
-anomaly-detection:
-	@echo "===> Building antrea/theia-anomaly-detection Docker image <==="
-	docker build --pull -t antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.anomaly-detection.ubuntu .
-	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) antrea/theia-anomaly-detection
-	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection
-	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION)
+# Theia currently supports two spark jobs, Throughput Anomaly Detection and Policy Recommendation.
+# This Dockerfile helps create unified dockerfile for both the spark jobs.
+.PHONY: spark-jobs
+spark-jobs:
+	@echo "===> Building antrea/theia-spark-jobs Docker image <==="
+	docker build --pull -t antrea/theia-spark-jobs:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.spark-jobs.ubuntu .
+	docker tag antrea/theia-spark-jobs:$(DOCKER_IMG_VERSION) antrea/theia-spark-jobs
+	docker tag antrea/theia-spark-jobs:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-spark-jobs
+	docker tag antrea/theia-spark-jobs:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-spark-jobs:$(DOCKER_IMG_VERSION)
 
 THEIA_BINARIES := theia-darwin theia-linux theia-windows
 $(THEIA_BINARIES): theia-%:

diff --git a/build/images/Dockerfile.anomaly-detection.ubuntu b/build/images/Dockerfile.anomaly-detection.ubuntu
diff --git a/...s/Dockerfile.policy-recommendation.ubuntu → build/images/Dockerfile.spark-jobs.ubuntu b/...s/Dockerfile.policy-recommendation.ubuntu → build/images/Dockerfile.spark-jobs.ubuntu
@@ -1,7 +1,7 @@
 FROM gcr.io/spark-operator/spark-py:v3.1.1
 
 LABEL maintainer="Antrea <[email protected]>"
-LABEL description="A docker image to deploy policy recommendation Spark job."
+LABEL description="A docker image to deploy policy recommendation and throughput anomaly detector Spark job."
 
 WORKDIR /opt/spark/work-dir
 USER root
@@ -10,10 +10,14 @@ RUN apt-get --allow-releaseinfo-change update && \
     apt-get install -y --no-install-recommends wget ca-certificates && \
     wget https://github.com/ClickHouse/clickhouse-jdbc/releases/download/v0.3.1/clickhouse-jdbc-0.3.1.jar -P /opt/spark/jars/
 
-RUN pip3 install --upgrade pip && \
-    pip3 install pyyaml && \
-    pip3 install kubernetes
-
 COPY plugins/policy-recommendation/policy_recommendation_job.py /opt/spark/work-dir/policy_recommendation_job.py
 COPY plugins/policy-recommendation/policy_recommendation_utils.py /opt/spark/work-dir/policy_recommendation_utils.py
 COPY plugins/policy-recommendation/antrea_crd.py /opt/spark/work-dir/antrea_crd.py
+COPY plugins/anomaly-detection/AnomalyDetection.py /opt/spark/work-dir/AnomalyDetection.py
+COPY plugins/anomaly-detection/requirements.txt /opt/spark/work-dir/requirements.txt
+
+
+RUN pip3 install --upgrade pip && \
+    pip3 install pyyaml && \
+    pip3 install kubernetes && \
+     pip3 install -r  /opt/spark/work-dir/requirements.txt
diff --git a/ci/clair-scan/run.sh b/ci/clair-scan/run.sh
@@ -43,8 +43,8 @@ VERSION=$(git ls-remote --tags --ref https://github.com/antrea-io/theia.git | \
 
 echo "Scanning Theia version $VERSION"
 
-docker pull "antrea/theia-policy-recommendation:$VERSION"
-docker pull "antrea/theia-policy-recommendation:latest"
+docker pull "antrea/theia-spark-jobs:$VERSION"
+docker pull "antrea/theia-spark-jobs:latest"
 
 echo "Downloading clair-scanner"
 curl -Lo ./clair-scanner "https://github.com/arminc/clair-scanner/releases/download/v12/clair-scanner_${ostype}_amd64"
@@ -78,12 +78,13 @@ else
     CLAIR_SCANNER_IP="$EN0_IP"
 fi
 
-./clair-scanner --clair=http://localhost:6060 --ip "$CLAIR_SCANNER_IP" -r "policy-recommendation.clair.$VERSION.json" "antrea/theia-policy-recommendation:$VERSION" || test -f "policy-recommendation.clair.$VERSION.json"
-./clair-scanner --clair=http://localhost:6060 --ip "$CLAIR_SCANNER_IP" -r "policy-recommendation.clair.latest.json" "antrea/theia-policy-recommendation:latest" || test -f "policy-recommendation.clair.latest.json"
+./clair-scanner --clair=http://localhost:6060 --ip "$CLAIR_SCANNER_IP" -r "spark-jobs.clair.$VERSION.json" "antrea/theia-spark-jobs:$VERSION" || test -f "spark-jobs.clair.$VERSION.json"
+./clair-scanner --clair=http://localhost:6060 --ip "$CLAIR_SCANNER_IP" -r "spark-jobs.clair.latest.json" "antrea/theia-spark-jobs:latest" || test -f "spark-jobs.clair.latest.json"
 
 if [ -n "$REPORTS_OUT_DIR" ]; then
     echo "Copying Clair scan reports to $REPORTS_OUT_DIR"
     cp *.clair.$VERSION.json *.clair.latest.json "$REPORTS_OUT_DIR"
 fi
 
-go run . --report "policy-recommendation.clair.$VERSION.json" --report-cmp "policy-recommendation.clair.latest.json"
+go run . --report "spark-jobs.clair.$VERSION.json" --report-cmp "spark-jobs.clair.latest.json"
+
diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh
@@ -366,9 +366,8 @@ function deliver_antrea {
     docker save -o theia-spark-operator.tar projects.registry.vmware.com/antrea/theia-spark-operator:v1beta2-1.3.3-3.1.1
     docker save -o theia-zookeeper.tar projects.registry.vmware.com/antrea/theia-zookeeper:3.8.0
 
-    (cd $GIT_CHECKOUT_DIR && make policy-recommendation && make clickhouse-monitor && make clickhouse-server && make theia-manager && make anomaly-detection)
-    docker save -o theia-policy-recommendation.tar projects.registry.vmware.com/antrea/theia-policy-recommendation:latest
-    docker save -o theia-anomaly-detection.tar projects.registry.vmware.com/antrea/theia-anomaly-detection:latest
+    (cd $GIT_CHECKOUT_DIR && make clickhouse-monitor && make clickhouse-server && make theia-manager && make spark-jobs)
+    docker save -o theia-spark-jobs.tar projects.registry.vmware.com/antrea/theia-spark-jobs:latest
     docker save -o theia-clickhouse-monitor.tar projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
     docker save -o theia-clickhouse-server.tar projects.registry.vmware.com/antrea/theia-clickhouse-server:latest
     docker save -o theia-manager.tar projects.registry.vmware.com/antrea/theia-manager:latest
@@ -394,8 +393,7 @@ function deliver_antrea {
         copy_image theia-metrics-exporter.tar projects.registry.vmware.com/antrea/theia-metrics-exporter  ${IPs[$i]} $image_tag true
         copy_image theia-zookeeper.tar projects.registry.vmware.com/antrea/theia-zookeeper  ${IPs[$i]} 3.8.0 true
         copy_image theia-spark-operator.tar projects.registry.vmware.com/antrea/theia-spark-operator ${IPs[$i]} v1beta2-1.3.3-3.1.1 true
-        copy_image theia-policy-recommendation.tar projects.registry.vmware.com/antrea/theia-policy-recommendation ${IPs[$i]} latest true
-        copy_image theia-anomaly-detection.tar projects.registry.vmware.com/antrea/theia-anomaly-detection ${IPs[$i]} latest true
+        copy_image theia-spark-jobs.tar projects.registry.vmware.com/antrea/theia-spark-jobs ${IPs[$i]} latest true
         copy_image theia-clickhouse-monitor.tar projects.registry.vmware.com/antrea/theia-clickhouse-monitor ${IPs[$i]} latest true
         copy_image theia-clickhouse-server.tar projects.registry.vmware.com/antrea/theia-clickhouse-server ${IPs[$i]} latest true
         copy_image theia-manager.tar projects.registry.vmware.com/antrea/theia-manager ${IPs[$i]} latest true

diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -122,11 +122,10 @@ for image in "${COMMON_IMAGES_LIST[@]}"; do
     done
 done
 
-COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/theia-policy-recommendation:latest"\
+COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/theia-spark-jobs:latest"\
                      "projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest"\
                      "projects.registry.vmware.com/antrea/theia-clickhouse-server:latest"\
-                     "projects.registry.vmware.com/antrea/theia-manager:latest"\
-                     "projects.registry.vmware.com/antrea/theia-anomaly-detection:latest")
+                     "projects.registry.vmware.com/antrea/theia-manager:latest")
 
 printf -v COMMON_IMAGES "%s " "${COMMON_IMAGES_LIST[@]}"