diff --git a/.github/actions/docker-build-image/action.yml b/.github/actions/docker-build-image/action.yml new file mode 100644 index 00000000000..d974fd78cf9 --- /dev/null +++ b/.github/actions/docker-build-image/action.yml @@ -0,0 +1,77 @@ +name: "Build docker image if tag is not 'latest'" + +description: "Build docker image if tag is not 'latest'" + +inputs: + docker-registry: + description: "Docker registry" + required: true + docker-login: + description: "Docker login" + required: true + docker-password: + description: "Docker password" + required: true + dockercontext: + description: "Docker build context" + required: false + default: "." + dockerfile: + description: "Dockerfile" + required: false + default: "Dockerfile" + image-name: + description: "Docker image name" + required: false + default: "${{ github.event.repository.name }}" + image-tag: + description: "Docker image tag" + required: false + default: "latest" + build-args: + description: "Docker build argunebts" + required: false + default: "" + +outputs: + docker-image: + description: "Docker image" + value: ${{ steps.image.outputs.value }} + +runs: + using: "composite" + steps: + - name: "Set docker image tag" + id: image + shell: bash + run: echo "value=${{ inputs.docker-registry }}/${{ inputs.image-name }}:${{ inputs.image-tag }}" >> $GITHUB_OUTPUT + + - name: "Set DOCKER_CONFIG because buildx stores data there" + if: inputs.image-tag != 'latest' + shell: bash + run: echo "DOCKER_CONFIG=./.docker" >> $GITHUB_ENV + + - name: "Set up Docker Buildx" + if: inputs.image-tag != 'latest' + uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: --debug + driver: docker + + - name: "Login to docker registry" + if: inputs.image-tag != 'latest' + uses: docker/login-action@v3 + with: + registry: ${{ inputs.docker-registry }} + username: ${{ inputs.docker-login }} + password: ${{ inputs.docker-password }} + + - name: "Build and push" + if: inputs.image-tag != 'latest' + uses: docker/build-push-action@v6 + with: + context: ${{ inputs.dockercontext }} + file: ${{ inputs.dockerfile }} + tags: ${{ steps.image.outputs.value }} + build-args: ${{ inputs.build-args }} + push: true diff --git a/.github/actions/docker-tag/action.yml b/.github/actions/docker-tag/action.yml new file mode 100644 index 00000000000..fde3e545d3f --- /dev/null +++ b/.github/actions/docker-tag/action.yml @@ -0,0 +1,54 @@ +name: "Docker image tag" + +description: "Get a docker image tag based on changes in a docker build context" + +inputs: + dockercontext: + description: "Docker build context" + required: false + default: "." + dockerfile: + description: "Dockerfile" + required: false + default: "Dockerfile" + dockerignore: + description: "Dockerignore file" + required: false + default: ".dockerignore" + defaulttag: + description: "A default docker image tag" + required: false + default: "latest" + +outputs: + tag: + description: "Docker image tag" + value: ${{ steps.tag.outputs.value }} + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - name: "Download '.dockerignore' file parser" + shell: bash + run: | + curl -L "https://github.com/johnstairs/dockerignore-filter/releases/download/v0.1.6/dockerignore-filter_Linux_x86_64" -o dockerignore-filter + chmod +x dockerignore-filter + + - name: "Get list of files from the docker build context (including Dockerfile and .dockerignore)" + shell: bash + run: echo "DOCKER_BUILD_CONTEXT_FILES=$(find ${{ inputs.dockercontext }} -type f | ./dockerignore-filter ${{ inputs.dockercontext }}/${{ inputs.dockerignore }} | tr '\n' ' ') ${{ inputs.dockercontext }}/${{ inputs.dockerfile }} ${{ inputs.dockercontext }}/${{ inputs.dockerignore }}" >> $GITHUB_ENV + + - name: "Get list of changes files in the docker build context" + shell: bash + run: | + git branch --delete --force ${{ github.event.repository.default_branch }} || true + git fetch --no-tags --force --prune --no-recurse-submodules --depth=1 origin ${{ github.event.repository.default_branch }}:refs/remotes/origin/${{ github.event.repository.default_branch }} + echo "DOCKER_BUILD_CONTEXT_FILES=$(git diff --name-only origin/${{ github.event.repository.default_branch }} -- $DOCKER_BUILD_CONTEXT_FILES | tr '\n' ' ')" >> $GITHUB_ENV + + - name: "Set a docker image tag" + id: tag + shell: bash + run: echo "value=$([[ ! -z \"$DOCKER_BUILD_CONTEXT_FILES\" ]] && echo $(git rev-parse --short HEAD) || echo ${{ inputs.defaulttag }})" >> $GITHUB_OUTPUT + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..2c2f812a7ee --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,11 @@ +name: CI pipeline + +on: + pull_request: + branches: [ develop ] + +jobs: + pipeline: + if: github.server_url != 'https://github.com' + uses: ./.github/workflows/pipeline.yml + secrets: inherit diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 00000000000..4d807174d27 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,101 @@ +name: CI Pipeline + +on: + workflow_call: + +jobs: + docker-tag: + name: Check if 'latest' tag could be used (no build docker images) + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - run: | + sudo apt update -qq + sudo apt install -y git curl jq sudo ca-certificates + sudo update-ca-certificates + - uses: actions/checkout@v4 + - uses: ./.github/actions/docker-tag + id: tag + with: + dockerfile: Jenkins/fast-release/Dockerfile.ci + + variants: + name: Define supported AIMET variants + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.final.outputs.value }} + steps: + - run: | + sudo apt update -qq + sudo apt install -y git curl jq sudo ca-certificates + sudo update-ca-certificates + + - name: Torch variants + run: | + set -x + VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [ + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":"" }, + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":"11.8.0" } + ]') + echo "VALUE=$VALUE" >> $GITHUB_ENV + + - name: Tensorflow variants + run: | + set -x + VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [ + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"", "VER_ONNX":"", "VER_CUDA":"" }, + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"", "VER_ONNX":"", "VER_CUDA":"11.8.0" } + ]') + echo "VALUE=$VALUE" >> $GITHUB_ENV + + - name: ONNX variants + run: | + set -x + VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [ + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":"" }, + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":"11.7.1" } + ]') + echo "VALUE=$VALUE" >> $GITHUB_ENV + + - name: (Last step) Generate few extra properties for each variant + id: final + run: | + set -x + VALUE=$(echo "$VALUE" | jq -c '.include[] |= . + { + "runs-on":(if .VER_CUDA != "" then "k8s-gpu" else "ubuntu-latest" end), + "id":("" + +(if .VER_TENSORFLOW != "" then "tf-" else "" end) + +(if .VER_ONNX != "" then "onnx-" elif .VER_TORCH != "" then "torch-" else "" end) + +(if .VER_CUDA != "" then "gpu" else "cpu" end) + ) + }') + echo "value=$VALUE" >> $GITHUB_OUTPUT + + docker-build-image: + name: Docker image ${{ matrix.id }} + runs-on: ubuntu-latest + needs: [docker-tag, variants] + strategy: + matrix: ${{ fromJSON(needs.variants.outputs.matrix) }} + steps: + - run: | + sudo apt update -qq + sudo apt install -y git curl jq sudo ca-certificates + sudo update-ca-certificates + - uses: actions/checkout@v4 + - uses: ./.github/actions/docker-build-image + with: + dockerfile: Jenkins/fast-release/Dockerfile.ci + docker-login: ${{ secrets.DOCKER_LOGIN }} + docker-password: ${{ secrets.DOCKER_CREDENTIALS }} + docker-registry: ${{ secrets.DOCKER_REGISTRY }} + image-name: "${{ secrets.DOCKER_IMAGE }}-${{ matrix.id }}" + image-tag: ${{ needs.docker-tag.outputs.tag }} + build-args: | + VER_PYTHON=${{ matrix.VER_PYTHON }} + VER_CUDA=${{ matrix.VER_CUDA }} + VER_TORCH=${{ matrix.VER_TORCH }} + VER_TENSORFLOW=${{ matrix.VER_TENSORFLOW }} + VER_ONNX=${{ matrix.VER_ONNX }} +