ci.yaml

# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: End-to-end tests

on:
  workflow_run:
    workflows: [image]
    types:
      - completed
    branches:
      - main

jobs:
  e2e-tests-nvidiadriver:
    runs-on: linux-amd64-cpu4

    steps:
    - name: Check out code
      uses: actions/checkout@v4

    - name: Set up Holodeck
      uses: NVIDIA/holodeck@v0.2.1
      env:
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
      with:
        aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
        holodeck_config: "tests/holodeck.yaml"
        
    - name: Get public dns name
      id: get_public_dns_name
      uses: mikefarah/yq@master
      with:
        cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml

    - name:  Set and Calculate test vars
      run: |
        echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
        echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
        echo "${{ secrets.AWS_SSH_KEY }}" > ${{ github.workspace }}/key.pem && chmod 400 ${{ github.workspace }}/key.pem
        echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
        DRIVER_VERSIONS=$(grep '^DRIVER_VERSIONS ?=' versions.mk | awk -F' ?= ' '{print $2}')
        echo "DRIVER_VERSIONS=$DRIVER_VERSIONS" >> $GITHUB_ENV
        echo "PRIVATE_REGISTRY=ghcr.io" >> $GITHUB_ENV

    - name: Validate gpu driver
      env:
        TEST_CASE: "./tests/cases/nvidia-driver.sh"
        GPU_OPERATOR_OPTIONS: "--set driver.repository=${{ env.PRIVATE_REGISTRY }}/nvidia"
      run: |
        rc=0
        for DRIVER_VERSION in ${DRIVER_VERSIONS}; do
          echo "Running e2e for DRIVER_VERSION=$DRIVER_VERSION"
          status=0
          OPERATOR_OPTIONS="${GPU_OPERATOR_OPTIONS} --set driver.version=${COMMIT_SHORT_SHA}-${DRIVER_VERSION}"
          # add escape character for space
          OPERATOR_OPTIONS=$(printf '%q ' "$OPERATOR_OPTIONS")
          ./tests/ci-run-e2e.sh "${TEST_CASE}" "${OPERATOR_OPTIONS}" || status=$?
          if [ $status -ne 0 ]; then
            echo "e2e validation failed for driver version $DRIVER_VERSION with status $status"
            rc=$status
          fi
        done
        ./tests/scripts/pull.sh /tmp/logs logs
        exit $rc

    - name: Archive test logs
      if: ${{ failure() }}
      uses: actions/upload-artifact@v4
      with:
        name: nvidiadriver-e2e-test-logs
        path: ./logs/
        retention-days: 15