From 52c902b52542405e4a9cba50d48b3465c6947cb2 Mon Sep 17 00:00:00 2001 From: Anton Belodedenko <2033996+ab77@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:29:32 -0700 Subject: [PATCH] update CI config --- .github/workflows/balena.yml | 197 ++++++++++++++++++++--------------- src/balena-tests | 2 +- 2 files changed, 114 insertions(+), 85 deletions(-) diff --git a/.github/workflows/balena.yml b/.github/workflows/balena.yml index 10679cc7a9..ad0c4224f5 100644 --- a/.github/workflows/balena.yml +++ b/.github/workflows/balena.yml @@ -9,19 +9,17 @@ on: env: # nested virtualisation not supported on AWS/EC2 instance types|classes other than X.metal - AWS_EC2_INSTANCE_TYPE: c6a.xlarge - AWS_EC2_LAUNCH_TEMPLATE: lt-011a66522ae6c5754 + AWS_EC2_INSTANCE_TYPE: c6a.2xlarge + AWS_EC2_LAUNCH_TEMPLATE: lt-02e10a4f66261319d AWS_EC2_LT_VERSION: 6 AWS_IAM_USERNAME: balena-tests-iam-User-1GXO3XP12N6LL AWS_REGION: us-east-1 - AWS_VPC_SECURITY_GROUP_IDS: sg-093fa6ade710210ab - AWS_VPC_SUBNET_ID: subnet-02d18a08ea4058574 - BALENA_CLI_URL: https://github.com/balena-io/balena-cli/releases/download - BALENA_CLI_VERSION: 13.10.0 + AWS_VPC_SECURITY_GROUP_IDS: sg-057937f4d89d9d51c + AWS_VPC_SUBNET_ID: 'subnet-02d18a08ea4058574 subnet-0a026eae1df907a09' # https://github.com/balena-io/balena-cli/issues/2447 DEBUG: 0 DEVICE_PROXY_TLD: balena-devices.com - DEVICE_TYPE: genericx86-64-ext + DEVICE_TYPE: generic-amd64 # https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns # https://github.com/balena-io/autohat/blob/master/resources/qemu.robot#L23 # https://github.com/balena-io-playground/balena-nested/blob/ab77/open-balena/guests.yml#L29-L32 @@ -29,10 +27,11 @@ env: ENVIRONMENT: balena-cloud.com FLEET: balena/open-balena OPENBALENA_TESTS_SERVICE: balena-tests - REGISTRY_USER: balenaci + # spot, on-demand + MARKET_TYPES: spot RELEASES: 50 RETRY: 3 - SOCAT_VERSION: 1.7.4.3 + SOCAT_VERSION: 1.7.4.4 VARIANT: prod VERBOSE: 'true' @@ -49,64 +48,78 @@ jobs: # FIXME: remove once balenaBlocks/balenaVirt is a thing submodules: true - # https://github.com/pdcastro/ssh-uuid#why - # https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others - - name: install additional dependencies - id: extra-dependencies - run: | - set -ue - - [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x + - uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0 + id: ephemeral + with: + app_id: ${{ vars.APP_ID || '291899' }} + installation_id: ${{ vars.INSTALLATION_ID || '34046749' }} + permissions: >- + { + "actions": "read", + "contents": "read", + "metadata": "read", + "packages": "read", + "pull_requests": "write" + } + private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} - pushd "${RUNNER_TEMP}" + - uses: aws-actions/configure-aws-credentials@v2 + with: + aws-region: ${{ vars.AWS_REGION || env.AWS_REGION || 'us-east-1' }} + role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} + # https://github.com/balena-io/environment-production/blob/master/aws/balenacloud/iam.yml + role-to-assume: ${{ vars.AWS_IAM_ROLE }} - sudo apt install -y \ - build-essential \ - git-secret \ - libreadline-dev \ - libssl-dev \ - libwrap0-dev \ - ssh + # https://github.com/pdcastro/ssh-uuid#why + # https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others + - name: install additional dependencies + id: extra-dependencies + shell: bash + run: | + set -ue + echo '::notice::install additional dependencies' - release_zip="balena-cli-v${BALENA_CLI_VERSION}-linux-x64-standalone.zip" + [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x - wget -q "${BALENA_CLI_URL}/v${BALENA_CLI_VERSION}/${release_zip}" \ - && unzip -q "${release_zip}" -d "${RUNNER_TEMP}" - "${RUNNER_TEMP}/balena-cli/balena" version + source exponential_backoff - "${RUNNER_TEMP}/balena-cli/balena" login --token '${{ secrets.BALENA_API_KEY_TEST }}' + mkdir -p "${RUNNER_TEMP}/ssh-uuid" - "${RUNNER_TEMP}/balena-cli/balena" whoami + wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \ + && chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \ + && ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid" - echo "${RUNNER_TEMP}/balena-cli" >> $GITHUB_PATH + with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}' - mkdir -p "${RUNNER_TEMP}/ssh-uuid" + balena version - wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \ - && chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \ - && ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid" + "${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help - "${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help + grep -q "${RUNNER_TEMP}/ssh-uuid" "${GITHUB_PATH}" \ + || echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}" - echo "${RUNNER_TEMP}/ssh-uuid" >> $GITHUB_PATH - curl --silent --retry ${{ env.RETRY }} --fail \ - http://www.dest-unreach.org/socat/download/socat-${SOCAT_VERSION}.tar.gz | tar -xzvf - \ - && cd socat-${SOCAT_VERSION} \ - && ./configure \ - && make \ - && sudo make install - - socat -V - - popd + - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + id: ephemeral + with: + app_id: ${{ vars.APP_ID || '291899' }} + installation_retrieval_mode: id + installation_retrieval_payload: ${{ vars.INSTALLATION_ID || '34046749' }} + permissions: >- + { + "actions": "read", + "contents": "read", + "metadata": "read", + "pull_requests": "write" + } + private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} - name: push draft or finalise release timeout-minutes: 60 id: push-release uses: balena-io/deploy-to-balena-action@master with: - balena_token: ${{ secrets.BALENA_API_KEY_PUSH }} + balena_token: ${{ secrets.BALENA_API_KEY }} cache: false environment: ${{ env.ENVIRONMENT }} fleet: ${{ env.FLEET }} @@ -115,9 +128,13 @@ jobs: versionbot: false registry_secrets: | { - "https://index.docker.io/v2/": { - "username": "${{ env.REGISTRY_USER }}", - "password": "${{ secrets.REGISTRY_PASS }}" + "ghcr.io": { + "username": "${{ github.actor }}", + "password": "${{ steps.ephemeral.outputs.token }}" + }, + "docker.io": { + "username": "${{ secrets.DOCKERHUB_USER }}", + "password": "${{ secrets.DOCKERHUB_TOKEN }}" } } @@ -129,7 +146,7 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena_device_uuid="$(openssl rand -hex 16)" @@ -160,8 +177,8 @@ jobs: balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}" done - echo "::set-output name=balena_device_uuid::${balena_device_uuid}" - echo "::set-output name=balena_device_id::${device_id}" + echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}" + echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}" # https://github.com/balena-io/balena-cli/issues/1543 - name: pin device to draft release @@ -172,7 +189,7 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena_releases="$(mktemp)" balena releases '${{ env.FLEET }}' | tail -n +2 | head -n ${{ env.RELEASES }} > "${balena_releases}" @@ -196,7 +213,7 @@ jobs: # pin DUT to draft release curl -X PATCH --silent --retry ${{ env.RETRY }} --fail -o /dev/null \ 'https://api.${{ env.ENVIRONMENT }}/v6/device?$filter=id%20in%20(${{ steps.register-test-device.outputs.balena_device_id }})' \ - -H 'authorization: Bearer ${{ secrets.BALENA_API_KEY_TEST }}' \ + -H 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \ -H 'content-type: application/json' \ --data-raw "{\"should_be_running__release\":${release_id}}" \ --compressed @@ -206,7 +223,7 @@ jobs: app_id="$(balena fleet ${{ env.FLEET }} | grep ^ID: | cut -c14-)" - echo "::set-output name=balena_app_id::${app_id}" + echo "balena_app_id=${app_id}" >> "${GITHUB_OUTPUT}" - name: configure test device environment id: configure-test-env @@ -216,7 +233,7 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena env add VERBOSE '${{ env.VERBOSE }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' @@ -299,10 +316,10 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' # cert-manager requires it to get whoami information for the user - balena env add API_TOKEN '${{ secrets.BALENA_API_KEY_TEST }}' \ + balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \ --service cert-manager \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' @@ -311,6 +328,15 @@ jobs: --service cert-manager \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' + # AWS credentials to backup/restore PKI assets + with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \ + --service cert-manager \ + --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' + + with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \ + --service cert-manager \ + --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' + - name: provision ephemeral test device id: provision-test-device if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} @@ -319,18 +345,21 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - for market_type in spot on-demand; do - # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html - response="$(aws ec2 run-instances \ - --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \ - --instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \ - $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \ - --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \ - --subnet-id '${{ env.AWS_VPC_SUBNET_ID }}' \ - --associate-public-ip-address \ - --user-data file://config.json \ - --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)" - + for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do + for market_type in ${{ env.MARKET_TYPES }}; do + # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html + response="$(aws ec2 run-instances \ + --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \ + --instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \ + $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \ + --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \ + --subnet-id "${subnet_id}" \ + --associate-public-ip-address \ + --user-data file://config.json \ + --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)" + + [[ -n $response ]] && break + done [[ -n $response ]] && break done @@ -342,7 +371,7 @@ jobs: aws ec2 wait instance-status-ok --instance-ids "${instance_id}" - echo "::set-output name=instance_id::${instance_id}" + echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}" env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -360,7 +389,7 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then ssh-keygen -N '' \ @@ -395,7 +424,7 @@ jobs: sleep "$(( (RANDOM % 5) + 5 ))s" done - echo "::set-output name=key_id::${GITHUB_SHA}" + echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" env: SSH_AUTH_SOCK: /tmp/ssh_agent.sock @@ -412,7 +441,7 @@ jobs: source functions - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena whoami && ssh-add -l @@ -434,13 +463,13 @@ jobs: source functions - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena whoami && ssh-add -l while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \ 'https://api.${{ env.ENVIRONMENT }}/supervisor/v1/device' \ - --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY_TEST }}' \ + --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \ --header 'Content-Type:application/json' \ --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \ --compressed | jq -r '.update_pending')" =~ ^true$ ]]; do @@ -487,7 +516,7 @@ jobs: source functions - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena whoami && ssh-add -l @@ -525,13 +554,13 @@ jobs: source functions - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena whoami && ssh-add -l while ! [[ $(curl -X POST --silent --retry ${{ env.RETRY }} --fail \ 'https://api.${{ env.ENVIRONMENT }}/supervisor/v2/applications/state' \ - --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY_TEST }}' \ + --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \ --header 'Content-Type:application/json' \ --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \ --compressed | jq -r '.[].services."${{ env.OPENBALENA_TESTS_SERVICE }}".status') =~ Run|run ]]; do @@ -558,7 +587,7 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \ | awk '{print $1}' | xargs balena key rm --yes || true @@ -575,7 +604,7 @@ jobs: [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x - balena login --token '${{ secrets.BALENA_API_KEY_TEST }}' + balena login --token '${{ secrets.BALENA_API_KEY }}' aws ec2 terminate-instances \ --instance-ids ${{ steps.provision-test-device.outputs.instance_id }} || true diff --git a/src/balena-tests b/src/balena-tests index a0ab0a3ff3..0581ea03b7 160000 --- a/src/balena-tests +++ b/src/balena-tests @@ -1 +1 @@ -Subproject commit a0ab0a3ff31ba2ac6ab9efd82f8ba4d92ccabf06 +Subproject commit 0581ea03b7c6e1bc0c26efe706ae3911809f03a6