Build docker images on a CI

Signed-off-by: Evgeny Mironov <[email protected]>

.github/actions/docker-build-image/action.yml

@@ -0,0 +1,77 @@
+name: "Build docker image if tag is not 'latest'"
+
+description: "Build docker image if tag is not 'latest'"
+
+inputs:
+  docker-registry:
+    description: "Docker registry"
+    required: true
+  docker-login:
+    description: "Docker login"
+    required: true
+  docker-password:
+    description: "Docker password"
+    required: true
+  dockercontext:
+    description: "Docker build context"
+    required: false
+    default: "."
+  dockerfile:
+    description: "Dockerfile"
+    required: false
+    default: "Dockerfile"
+  image-name:
+    description: "Docker image name"
+    required: false
+    default: "${{ github.event.repository.name }}"
+  image-tag:
+    description: "Docker image tag"
+    required: false
+    default: "latest"
+  build-args:
+    description: "Docker build argunebts"
+    required: false
+    default: ""
+
+outputs:
+  docker-image:
+    description: "Docker image"
+    value: ${{ steps.image.outputs.value }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Set docker image tag"
+      id: image
+      shell: bash
+      run: echo "value=${{ inputs.docker-registry }}/${{ inputs.image-name }}:${{ inputs.image-tag }}" >> $GITHUB_OUTPUT
+
+    - name: "Set DOCKER_CONFIG because buildx stores data there"
+      if: inputs.image-tag != 'latest'
+      shell: bash
+      run: echo "DOCKER_CONFIG=./.docker" >> $GITHUB_ENV
+
+    - name: "Set up Docker Buildx"
+      if: inputs.image-tag != 'latest'
+      uses: docker/setup-buildx-action@v3
+      with:
+        buildkitd-flags: --debug
+        driver: docker
+
+    - name: "Login to docker registry"
+      if: inputs.image-tag != 'latest'
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ inputs.docker-registry }}
+        username: ${{ inputs.docker-login }}
+        password: ${{ inputs.docker-password }}
+
+    - name: "Build and push"
+      if: inputs.image-tag != 'latest'
+      uses: docker/build-push-action@v6
+      with:
+        context: ${{ inputs.dockercontext }}
+        file: ${{ inputs.dockerfile }}
+        tags: ${{ steps.image.outputs.value }}
+        build-args: ${{ inputs.build-args }}
+        push: true

.github/actions/docker-tag/action.yml

@@ -0,0 +1,54 @@
+name: "Docker image tag"
+
+description: "Get a docker image tag based on changes in a docker build context"
+
+inputs:
+  dockercontext:
+    description: "Docker build context"
+    required: false
+    default: "."
+  dockerfile:
+    description: "Dockerfile"
+    required: false
+    default: "Dockerfile"
+  dockerignore:
+    description: "Dockerignore file"
+    required: false
+    default: ".dockerignore"
+  defaulttag:
+    description: "A default docker image tag"
+    required: false
+    default: "latest"
+
+outputs:
+  tag:
+    description: "Docker image tag"
+    value: ${{ steps.tag.outputs.value }}
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+
+    - name: "Download '.dockerignore' file parser"
+      shell: bash
+      run: |
+        curl -L "https://github.com/johnstairs/dockerignore-filter/releases/download/v0.1.6/dockerignore-filter_Linux_x86_64" -o dockerignore-filter
+        chmod +x dockerignore-filter
+
+    - name: "Get list of files from the docker build context (including Dockerfile and .dockerignore)"
+      shell: bash
+      run: echo "DOCKER_BUILD_CONTEXT_FILES=$(find ${{ inputs.dockercontext }} -type f | ./dockerignore-filter ${{ inputs.dockercontext }}/${{ inputs.dockerignore }} | tr '\n' ' ') ${{ inputs.dockercontext }}/${{ inputs.dockerfile }} ${{ inputs.dockercontext }}/${{ inputs.dockerignore }}" >> $GITHUB_ENV
+
+    - name: "Get list of changes files in the docker build context"
+      shell: bash
+      run: |
+        git branch --delete --force ${{ github.event.repository.default_branch }} || true
+        git fetch --no-tags --force --prune --no-recurse-submodules --depth=1 origin ${{ github.event.repository.default_branch }}:refs/remotes/origin/${{ github.event.repository.default_branch }}
+        echo "DOCKER_BUILD_CONTEXT_FILES=$(git diff --name-only origin/${{ github.event.repository.default_branch }} -- $DOCKER_BUILD_CONTEXT_FILES | tr '\n' ' ')" >> $GITHUB_ENV
+
+    - name: "Set a docker image tag"
+      id: tag
+      shell: bash
+      run: echo "value=$([[ ! -z \"$DOCKER_BUILD_CONTEXT_FILES\" ]] && echo $(git rev-parse --short HEAD) || echo ${{ inputs.defaulttag }})" >> $GITHUB_OUTPUT
+

.github/workflows/ci.yml

@@ -0,0 +1,11 @@
+name: CI pipeline
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  pipeline:
+    if: github.server_url != 'https://github.com'
+    uses: ./.github/workflows/pipeline.yml
+    secrets: inherit

.github/workflows/pipeline.yml

@@ -0,0 +1,101 @@
+name: CI Pipeline
+
+on:
+  workflow_call:
+
+jobs:
+  docker-tag:
+    name: Check if 'latest' tag could be used (no build docker images)
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+    steps:
+      - run: |
+          sudo apt update -qq
+          sudo apt install -y git curl jq sudo ca-certificates
+          sudo update-ca-certificates
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/docker-tag
+        id: tag
+        with:
+          dockerfile: Jenkins/fast-release/Dockerfile.ci
+
+  variants:
+    name: Define supported AIMET variants
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.final.outputs.value }}
+    steps:
+      - run: |
+          sudo apt update -qq
+          sudo apt install -y git curl jq sudo ca-certificates
+          sudo update-ca-certificates
+
+      - name: Torch variants
+        run: |
+          set -x
+          VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":""       },
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":"11.8.0" }
+            ]')
+          echo "VALUE=$VALUE" >> $GITHUB_ENV
+
+      - name: Tensorflow variants
+        run: |
+          set -x
+          VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"",      "VER_ONNX":"", "VER_CUDA":"" },
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"",      "VER_ONNX":"", "VER_CUDA":"11.8.0" }
+            ]')
+          echo "VALUE=$VALUE" >> $GITHUB_ENV
+
+      - name: ONNX variants
+        run: |
+          set -x
+          VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":""       },
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":"11.7.1" }
+            ]')
+          echo "VALUE=$VALUE" >> $GITHUB_ENV
+
+      - name: (Last step) Generate few extra properties for each variant
+        id: final
+        run: |
+          set -x
+          VALUE=$(echo "$VALUE" | jq -c '.include[] |= . + {
+            "runs-on":(if .VER_CUDA != "" then "k8s-gpu" else "ubuntu-latest" end),
+            "id":(""
+                    +(if .VER_TENSORFLOW != "" then "tf-" else "" end)
+                    +(if .VER_ONNX != "" then "onnx-" elif .VER_TORCH != "" then "torch-" else "" end)
+                    +(if .VER_CUDA != "" then "gpu" else "cpu" end)
+                 )
+            }')
+          echo "value=$VALUE" >> $GITHUB_OUTPUT
+
+  docker-build-image:
+    name: Docker image ${{ matrix.id }}
+    runs-on: ubuntu-latest
+    needs: [docker-tag, variants]
+    strategy:
+      matrix: ${{ fromJSON(needs.variants.outputs.matrix) }}
+    steps:
+      - run: |
+          sudo apt update -qq
+          sudo apt install -y git curl jq sudo ca-certificates
+          sudo update-ca-certificates
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/docker-build-image
+        with:
+          dockerfile: Jenkins/fast-release/Dockerfile.ci
+          docker-login: ${{ secrets.DOCKER_LOGIN }}
+          docker-password: ${{ secrets.DOCKER_CREDENTIALS }}
+          docker-registry: ${{ secrets.DOCKER_REGISTRY }}
+          image-name: "${{ secrets.DOCKER_IMAGE }}-${{ matrix.id }}"
+          image-tag: ${{ needs.docker-tag.outputs.tag }}
+          build-args: |
+            VER_PYTHON=${{ matrix.VER_PYTHON }}
+            VER_CUDA=${{ matrix.VER_CUDA }}
+            VER_TORCH=${{ matrix.VER_TORCH }}
+            VER_TENSORFLOW=${{ matrix.VER_TENSORFLOW }}
+            VER_ONNX=${{ matrix.VER_ONNX }}
+