Build Docker & cloud images #51
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Docker & cloud images | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| image_version: | |
| description: "Docker image version" | |
| required: true | |
| staging: | |
| description: "Staging build" | |
| type: boolean | |
| default: false | |
| build_docker: | |
| description: "Build docker images" | |
| type: boolean | |
| default: true | |
| build_aws: | |
| description: "Build AWS images" | |
| type: boolean | |
| default: true | |
| build_azure: | |
| description: "Build Azure images" | |
| type: boolean | |
| default: true | |
| build_gcp: | |
| description: "Build GCP images" | |
| type: boolean | |
| default: true | |
| build_oci: | |
| description: "Build OCI images" | |
| type: boolean | |
| default: true | |
| build_nebius: | |
| description: "Build Nebius images" | |
| type: boolean | |
| default: true | |
| env: | |
| PACKER_VERSION: "1.9.2" | |
| BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }} # staging ? prefix : '' | |
| jobs: | |
| build-docker: | |
| if: inputs.build_docker | |
| defaults: | |
| run: | |
| working-directory: docker | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Build and upload to DockerHub | |
| run: | | |
| docker buildx build --platform linux/arm64/v8 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-arm64 -f cpu/Dockerfile . | |
| docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-amd64 -f cpu/Dockerfile . | |
| docker manifest create dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }} --amend dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-arm64 --amend dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-amd64 | |
| docker manifest push dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }} | |
| docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-12.1 -f cuda/Dockerfile . | |
| build-aws-images: | |
| needs: build-docker | |
| if: always() && inputs.build_aws && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| defaults: | |
| run: | |
| working-directory: scripts/packer | |
| runs-on: ubuntu-latest | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| strategy: | |
| matrix: | |
| variant: [ "", "-cuda" ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download packer | |
| run: | | |
| wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| chmod +x packer | |
| - name: Run packer | |
| run: | | |
| ./packer build -var-file=versions.json $PROD_VARS -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX aws-image${{ matrix.variant }}.json | |
| env: | |
| PROD_VARS: ${{ !inputs.staging && '-var-file=aws-vars-prod.json' || '' }} # production ? var-file : '' | |
| build-azure-images: | |
| needs: build-docker | |
| if: always() && inputs.build_azure && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| defaults: | |
| run: | |
| working-directory: scripts/packer | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| VERSION: ${{ github.run_number }} | |
| strategy: | |
| matrix: | |
| variant: [ "", "-cuda", "-grid" ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: Azure/login@v1 | |
| name: Log in to az | |
| with: | |
| creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}' | |
| - name: Download packer | |
| run: | | |
| wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| chmod +x packer | |
| - name: Run packer | |
| run: | | |
| ./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX azure-image${{ matrix.variant }}.json | |
| - name: Publish azure image | |
| if: ${{ !inputs.staging }} | |
| run: | | |
| IMAGE_DEFINITION=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} | |
| IMAGE_NAME=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} | |
| ../publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME | |
| build-gcp-images: | |
| needs: build-docker | |
| if: always() && inputs.build_gcp && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| defaults: | |
| run: | |
| working-directory: scripts/packer | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| variant: [ "", "-cuda" ] | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider' | |
| service_account: '[email protected]' | |
| create_credentials_file: true | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Download packer | |
| run: | | |
| wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| chmod +x packer | |
| - name: Run packer | |
| run: | | |
| ./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX gcp-image${{ matrix.variant }}.json | |
| - name: Publish images | |
| run: | | |
| IMAGE_VERSION=${IMAGE_VERSION//./-} | |
| gcloud compute images add-iam-policy-binding ${BUILD_PREFIX}dstack${{ matrix.variant }}-$IMAGE_VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' | |
| env: | |
| IMAGE_VERSION: ${{ inputs.image_version }} | |
| build-oci-images: | |
| needs: build-docker | |
| if: always() && inputs.build_oci && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| env: | |
| OCI_COMPARTMENT: ocid1.compartment.oc1..aaaaaaaaxu2uq64unfa2imwkp37icxqv6f7gwp2mczdt2mukuqbkauwqmbtq | |
| OCI_SUBNET: ocid1.subnet.oc1.eu-frankfurt-1.aaaaaaaaewxkaqsmbi2tig5sfw4eexzo3mkb4zrpm4gwvfhqdddnxicxe4fa | |
| OCI_AVAILABILITY_DOMAIN: kZql:EU-FRANKFURT-1-AD-3 | |
| OCI_REGION: eu-frankfurt-1 | |
| strategy: | |
| matrix: | |
| variant: [ "", "-cuda" ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup OCI config | |
| run: | | |
| mkdir ~/.oci | |
| echo "$OCI_KEY_CONTENT" > ~/.oci/key.pem | |
| echo [DEFAULT] > ~/.oci/config | |
| echo region=$OCI_REGION >> ~/.oci/config | |
| echo tenancy=$OCI_TENANCY >> ~/.oci/config | |
| echo user=$OCI_USER >> ~/.oci/config | |
| echo fingerprint=$OCI_FINGERPRINT >> ~/.oci/config | |
| echo key_file=~/.oci/key.pem >> ~/.oci/config | |
| env: | |
| OCI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} | |
| OCI_USER: ${{ secrets.OCI_CLI_USER }} | |
| OCI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} | |
| OCI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} | |
| - name: Install packer | |
| working-directory: scripts/packer | |
| run: | | |
| wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| chmod +x packer | |
| ./packer plugins install github.com/hashicorp/oracle | |
| - name: Run packer | |
| working-directory: scripts/packer | |
| run: | | |
| ./packer build \ | |
| -var-file=versions.json \ | |
| -var image_version=${{ inputs.image_version }} \ | |
| -var build_prefix=$BUILD_PREFIX \ | |
| -var oci_compartment_ocid=$OCI_COMPARTMENT \ | |
| -var oci_subnet_ocid=$OCI_SUBNET \ | |
| -var oci_availability_domain=$OCI_AVAILABILITY_DOMAIN \ | |
| oci-image${{ matrix.variant }}.json | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies for publishing | |
| run: | | |
| pip install .[oci] | |
| - name: Copy image to target regions | |
| run: | | |
| python scripts/oci_image_tools.py copy \ | |
| --image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \ | |
| --from $OCI_REGION \ | |
| --compartment $OCI_COMPARTMENT | |
| - name: Publish image in OCI Marketplace | |
| run: | | |
| python scripts/oci_image_tools.py publish \ | |
| --image ${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }} \ | |
| --compartment $OCI_COMPARTMENT \ | |
| --version ${{ inputs.image_version }} \ | |
| --description "Image for running workloads with dstack - https://dstack.ai/" \ | |
| --os "Ubuntu 22.04" \ | |
| --contact-name dstack \ | |
| --contact-email [email protected] | |
| build-nebius-images: | |
| needs: build-docker | |
| if: always() && inputs.build_nebius && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| defaults: | |
| run: | |
| working-directory: scripts/packer | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Get Nebius CLI | |
| run: | | |
| echo "CLI_VERSION=$CLI_VERSION" | |
| curl -sSL https://storage.ai.nebius.cloud/ncp/install.sh | bash | |
| echo "$HOME/nebius-cloud/bin" >> $GITHUB_PATH | |
| env: | |
| CLI_VERSION: 0.113.0+Nebius-AI | |
| - name: Write Nebius credentials | |
| uses: jsdaniell/[email protected] | |
| with: | |
| name: "service_account.json" | |
| json: ${{ secrets.NEBIUS_SERVICE_ACCOUNT }} | |
| dir: "scripts/packer/" | |
| - name: Setup Nebius profile | |
| run: | | |
| ncp config profile create packer | |
| ncp config set endpoint api.ai.nebius.cloud:443 | |
| ncp config set service-account-key service_account.json | |
| rm service_account.json | |
| - name: Download packer | |
| run: | | |
| wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip | |
| chmod +x packer | |
| ./packer init . | |
| - name: Run packer (HCL2) | |
| run: | | |
| export PKR_VAR_nebius_token=$(ncp iam create-token) | |
| ./packer build -only yandex.nebius,yandex.nebius-cuda -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX . | |
| env: | |
| PKR_VAR_nebius_folder_id: ${{ secrets.NEBIUS_FOLDER_ID }} | |
| PKR_VAR_nebius_subnet_id: ${{ secrets.NEBIUS_SUBNET_ID }} |