Skip to content

Commit

Permalink
update CI config
Browse files Browse the repository at this point in the history
  • Loading branch information
@ab77
ab77 committed Apr 22, 2024
1 parent 8613284 commit 52c902b
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 85 deletions.
197 changes: 113 additions & 84 deletions .github/workflows/balena.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,29 @@ on:

env:
# nested virtualisation not supported on AWS/EC2 instance types|classes other than X.metal
AWS_EC2_INSTANCE_TYPE: c6a.xlarge
AWS_EC2_LAUNCH_TEMPLATE: lt-011a66522ae6c5754
AWS_EC2_INSTANCE_TYPE: c6a.2xlarge
AWS_EC2_LAUNCH_TEMPLATE: lt-02e10a4f66261319d
AWS_EC2_LT_VERSION: 6
AWS_IAM_USERNAME: balena-tests-iam-User-1GXO3XP12N6LL
AWS_REGION: us-east-1
AWS_VPC_SECURITY_GROUP_IDS: sg-093fa6ade710210ab
AWS_VPC_SUBNET_ID: subnet-02d18a08ea4058574
BALENA_CLI_URL: https://github.com/balena-io/balena-cli/releases/download
BALENA_CLI_VERSION: 13.10.0
AWS_VPC_SECURITY_GROUP_IDS: sg-057937f4d89d9d51c
AWS_VPC_SUBNET_ID: 'subnet-02d18a08ea4058574 subnet-0a026eae1df907a09'
# https://github.com/balena-io/balena-cli/issues/2447
DEBUG: 0
DEVICE_PROXY_TLD: balena-devices.com
DEVICE_TYPE: genericx86-64-ext
DEVICE_TYPE: generic-amd64
# https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns
# https://github.com/balena-io/autohat/blob/master/resources/qemu.robot#L23
# https://github.com/balena-io-playground/balena-nested/blob/ab77/open-balena/guests.yml#L29-L32
DNS_TLD: auto.balena-devices.com
ENVIRONMENT: balena-cloud.com
FLEET: balena/open-balena
OPENBALENA_TESTS_SERVICE: balena-tests
REGISTRY_USER: balenaci
# spot, on-demand
MARKET_TYPES: spot
RELEASES: 50
RETRY: 3
SOCAT_VERSION: 1.7.4.3
SOCAT_VERSION: 1.7.4.4
VARIANT: prod
VERBOSE: 'true'

Expand All @@ -49,64 +48,78 @@ jobs:
# FIXME: remove once balenaBlocks/balenaVirt is a thing
submodules: true

# https://github.com/pdcastro/ssh-uuid#why
# https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others
- name: install additional dependencies
id: extra-dependencies
run: |
set -ue
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
- uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
id: ephemeral
with:
app_id: ${{ vars.APP_ID || '291899' }}
installation_id: ${{ vars.INSTALLATION_ID || '34046749' }}
permissions: >-
{
"actions": "read",
"contents": "read",
"metadata": "read",
"packages": "read",
"pull_requests": "write"
}
private_key: ${{ secrets.GH_APP_PRIVATE_KEY }}

pushd "${RUNNER_TEMP}"
- uses: aws-actions/configure-aws-credentials@v2
with:
aws-region: ${{ vars.AWS_REGION || env.AWS_REGION || 'us-east-1' }}
role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
# https://github.com/balena-io/environment-production/blob/master/aws/balenacloud/iam.yml
role-to-assume: ${{ vars.AWS_IAM_ROLE }}

sudo apt install -y \
build-essential \
git-secret \
libreadline-dev \
libssl-dev \
libwrap0-dev \
ssh
# https://github.com/pdcastro/ssh-uuid#why
# https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others
- name: install additional dependencies
id: extra-dependencies
shell: bash
run: |
set -ue
echo '::notice::install additional dependencies'
release_zip="balena-cli-v${BALENA_CLI_VERSION}-linux-x64-standalone.zip"
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
wget -q "${BALENA_CLI_URL}/v${BALENA_CLI_VERSION}/${release_zip}" \
&& unzip -q "${release_zip}" -d "${RUNNER_TEMP}"
"${RUNNER_TEMP}/balena-cli/balena" version
source exponential_backoff
"${RUNNER_TEMP}/balena-cli/balena" login --token '${{ secrets.BALENA_API_KEY_TEST }}'
mkdir -p "${RUNNER_TEMP}/ssh-uuid"
"${RUNNER_TEMP}/balena-cli/balena" whoami
wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \
&& chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \
&& ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid"
echo "${RUNNER_TEMP}/balena-cli" >> $GITHUB_PATH
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
mkdir -p "${RUNNER_TEMP}/ssh-uuid"
balena version
wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \
&& chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \
&& ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid"
"${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help
"${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help
grep -q "${RUNNER_TEMP}/ssh-uuid" "${GITHUB_PATH}" \
|| echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}"
echo "${RUNNER_TEMP}/ssh-uuid" >> $GITHUB_PATH
curl --silent --retry ${{ env.RETRY }} --fail \
http://www.dest-unreach.org/socat/download/socat-${SOCAT_VERSION}.tar.gz | tar -xzvf - \
&& cd socat-${SOCAT_VERSION} \
&& ./configure \
&& make \
&& sudo make install
socat -V
popd
- uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
id: ephemeral
with:
app_id: ${{ vars.APP_ID || '291899' }}
installation_retrieval_mode: id
installation_retrieval_payload: ${{ vars.INSTALLATION_ID || '34046749' }}
permissions: >-
{
"actions": "read",
"contents": "read",
"metadata": "read",
"pull_requests": "write"
}
private_key: ${{ secrets.GH_APP_PRIVATE_KEY }}

- name: push draft or finalise release
timeout-minutes: 60
id: push-release
uses: balena-io/deploy-to-balena-action@master
with:
balena_token: ${{ secrets.BALENA_API_KEY_PUSH }}
balena_token: ${{ secrets.BALENA_API_KEY }}
cache: false
environment: ${{ env.ENVIRONMENT }}
fleet: ${{ env.FLEET }}
Expand All @@ -115,9 +128,13 @@ jobs:
versionbot: false
registry_secrets: |
{
"https://index.docker.io/v2/": {
"username": "${{ env.REGISTRY_USER }}",
"password": "${{ secrets.REGISTRY_PASS }}"
"ghcr.io": {
"username": "${{ github.actor }}",
"password": "${{ steps.ephemeral.outputs.token }}"
},
"docker.io": {
"username": "${{ secrets.DOCKERHUB_USER }}",
"password": "${{ secrets.DOCKERHUB_TOKEN }}"
}
}
Expand All @@ -129,7 +146,7 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena_device_uuid="$(openssl rand -hex 16)"
Expand Down Expand Up @@ -160,8 +177,8 @@ jobs:
balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
done
echo "::set-output name=balena_device_uuid::${balena_device_uuid}"
echo "::set-output name=balena_device_id::${device_id}"
echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}"
# https://github.com/balena-io/balena-cli/issues/1543
- name: pin device to draft release
Expand All @@ -172,7 +189,7 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena_releases="$(mktemp)"
balena releases '${{ env.FLEET }}' | tail -n +2 | head -n ${{ env.RELEASES }} > "${balena_releases}"
Expand All @@ -196,7 +213,7 @@ jobs:
# pin DUT to draft release
curl -X PATCH --silent --retry ${{ env.RETRY }} --fail -o /dev/null \
'https://api.${{ env.ENVIRONMENT }}/v6/device?$filter=id%20in%20(${{ steps.register-test-device.outputs.balena_device_id }})' \
-H 'authorization: Bearer ${{ secrets.BALENA_API_KEY_TEST }}' \
-H 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
-H 'content-type: application/json' \
--data-raw "{\"should_be_running__release\":${release_id}}" \
--compressed
Expand All @@ -206,7 +223,7 @@ jobs:
app_id="$(balena fleet ${{ env.FLEET }} | grep ^ID: | cut -c14-)"
echo "::set-output name=balena_app_id::${app_id}"
echo "balena_app_id=${app_id}" >> "${GITHUB_OUTPUT}"
- name: configure test device environment
id: configure-test-env
Expand All @@ -216,7 +233,7 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena env add VERBOSE '${{ env.VERBOSE }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
Expand Down Expand Up @@ -299,10 +316,10 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
# cert-manager requires it to get whoami information for the user
balena env add API_TOKEN '${{ secrets.BALENA_API_KEY_TEST }}' \
balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
Expand All @@ -311,6 +328,15 @@ jobs:
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# AWS credentials to backup/restore PKI assets
with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
- name: provision ephemeral test device
id: provision-test-device
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
Expand All @@ -319,18 +345,21 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
for market_type in spot on-demand; do
# https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
response="$(aws ec2 run-instances \
--launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
--instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \
$([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
--security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
--subnet-id '${{ env.AWS_VPC_SUBNET_ID }}' \
--associate-public-ip-address \
--user-data file://config.json \
--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)"
for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
for market_type in ${{ env.MARKET_TYPES }}; do
# https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
response="$(aws ec2 run-instances \
--launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
--instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \
$([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
--security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
--subnet-id "${subnet_id}" \
--associate-public-ip-address \
--user-data file://config.json \
--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)"
[[ -n $response ]] && break
done
[[ -n $response ]] && break
done
Expand All @@ -342,7 +371,7 @@ jobs:
aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
echo "::set-output name=instance_id::${instance_id}"
echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}"
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
Expand All @@ -360,7 +389,7 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
ssh-keygen -N '' \
Expand Down Expand Up @@ -395,7 +424,7 @@ jobs:
sleep "$(( (RANDOM % 5) + 5 ))s"
done
echo "::set-output name=key_id::${GITHUB_SHA}"
echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
Expand All @@ -412,7 +441,7 @@ jobs:
source functions
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena whoami && ssh-add -l
Expand All @@ -434,13 +463,13 @@ jobs:
source functions
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena whoami && ssh-add -l
while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
'https://api.${{ env.ENVIRONMENT }}/supervisor/v1/device' \
--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY_TEST }}' \
--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
--header 'Content-Type:application/json' \
--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
--compressed | jq -r '.update_pending')" =~ ^true$ ]]; do
Expand Down Expand Up @@ -487,7 +516,7 @@ jobs:
source functions
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena whoami && ssh-add -l
Expand Down Expand Up @@ -525,13 +554,13 @@ jobs:
source functions
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena whoami && ssh-add -l
while ! [[ $(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
'https://api.${{ env.ENVIRONMENT }}/supervisor/v2/applications/state' \
--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY_TEST }}' \
--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
--header 'Content-Type:application/json' \
--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
--compressed | jq -r '.[].services."${{ env.OPENBALENA_TESTS_SERVICE }}".status') =~ Run|run ]]; do
Expand All @@ -558,7 +587,7 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \
| awk '{print $1}' | xargs balena key rm --yes || true
Expand All @@ -575,7 +604,7 @@ jobs:
[[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
balena login --token '${{ secrets.BALENA_API_KEY_TEST }}'
balena login --token '${{ secrets.BALENA_API_KEY }}'
aws ec2 terminate-instances \
--instance-ids ${{ steps.provision-test-device.outputs.instance_id }} || true
Expand Down
2 changes: 1 addition & 1 deletion src/balena-tests
Submodule balena-tests updated from a0ab0a to 0581ea

0 comments on commit 52c902b

Please sign in to comment.