Skip to content

Commit

Permalink
Initial PR /publish workflow into pre-merge /build-connector and …
Browse files Browse the repository at this point in the history 
…post-merge `/publish-connector` workflows (#12405)
  • Loading branch information
@noahkawasaki-airbyte
noahkawasaki-airbyte authored Apr 29, 2022
1 parent 1642630 commit e8406a2
Show file tree
Hide file tree
Showing 3 changed files with 584 additions and 2 deletions.
260 changes: 260 additions & 0 deletions .github/workflows/build-connector-command.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
name: Bump, Build, Test Connectors [EXPERIMENTAL]
on:
workflow_dispatch:
inputs:
repo:
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
required: false
default: "airbytehq/airbyte"
gitref:
description: "The git ref to check out from the specified repository."
required: false
default: master
connector:
description: "Airbyte Connector"
required: true
bump-version:
description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually"
required: false
default: "false"
run-tests:
description: "Should run tests"
required: false
default: "true"
comment-id:
description: "The comment-id of the slash command. Used to update the comment with the status."
required: false

jobs:
find_valid_pat:
name: "Find a PAT with room for actions"
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
pat: ${{ steps.variables.outputs.pat }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Check PAT rate limits
id: variables
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.AIRBYTEIO_PAT }} \
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
${{ secrets.SUPERTOPHER_PAT }} \
${{ secrets.DAVINCHIA_PAT }}
## Gradle Build
# In case of self-hosted EC2 errors, remove this block.
start-bump-build-test-connector-runner:
name: Start Build EC2 Runner
runs-on: ubuntu-latest
needs: find_valid_pat
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ needs.find_valid_pat.outputs.pat }}
# 80 gb disk
ec2-image-id: ami-0d648081937c75a73
bump-build-test-connector:
name: Bump, Build, Test Connector
needs: start-bump-build-test-connector-runner
runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
environment: more-secrets
steps:
############################
## SET UP ##
############################
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v0
with:
service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
export_default_credentials: true
- name: Search for valid connector name format
id: regex
uses: AsasInnab/regex-action@v1
with:
regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$"
regex_flags: "i" # required to be set for this plugin
search_string: ${{ github.event.inputs.connector }}
- name: Validate input workflow format
if: steps.regex.outputs.first_match != github.event.inputs.connector
run: echo "The connector provided has an invalid format!" && exit 1
- name: Link comment to workflow run
if: github.event.inputs.comment-id
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
- name: Checkout Airbyte
uses: actions/checkout@v2
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
token: ${{ secrets.OCTAVIA_PAT }}
- name: Install Java
uses: actions/setup-java@v1
with:
java-version: "17"
- name: Install Python
uses: actions/setup-python@v2
with:
python-version: "3.9"
- name: Install Pyenv and Tox
run: |
python3 -m pip install --quiet virtualenv==16.7.9 --user
python3 -m virtualenv venv
source venv/bin/activate
pip install --quiet tox==3.24.4
- name: Install yq
if: github.event.inputs.bump-version != 'false' && success()
run: |
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
sudo add-apt-repository ppa:rmescandon/yq
sudo apt update
sudo apt install yq -y
- name: Test and install CI scripts
# all CI python packages have the prefix "ci_"
run: |
source venv/bin/activate
tox -r -c ./tools/tox_ci.ini
pip install --quiet -e ./tools/ci_*
- name: Get Credentials for ${{ github.event.inputs.connector }}
run: |
source venv/bin/activate
ci_credentials ${{ github.event.inputs.connector }}
env:
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
# TODO: seems like this should run in post-merge workflow
# - name: Prepare Sentry
# if: startsWith(github.event.inputs.connector, 'connectors')
# run: |
# curl -sL https://sentry.io/get-cli/ | bash
# - name: Create Sentry Release
# if: startsWith(github.event.inputs.connector, 'connectors')
# run: |
# sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing
# env:
# SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
# SENTRY_ORG: airbyte-5j
# SENTRY_PROJECT: airbyte-connectors
############################
## BUMP ##
############################
- name: Bump Connector Version
if: github.event.inputs.bump-version != 'false' && success()
run: ./tools/integrations/manage.sh bump_version airbyte-integrations/${{ github.event.inputs.connector }}
- name: Commit and Push Version Bump
if: github.event.inputs.bump-version != 'false' && success()
run: |
git config user.name 'Octavia Squidington III'
git config user.email '[email protected]'
git add -u
git commit -m "bump-version ${{github.event.inputs.connector}}"
git push origin ${{ github.event.inputs.gitref }}
- name: Add Version Bump Success Comment
if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :rocket: Bumped version for ${{github.event.inputs.connector}}
- name: Add Version Bump Failure Comment
if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && !success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :x: Couldn't bump version for ${{github.event.inputs.connector}}
############################
## BUILD AND TEST ##
############################
- name: Build ${{ github.event.inputs.connector }}
run: ./tools/integrations/manage.sh build_experiment airbyte-integrations/${{ github.event.inputs.connector }}
id: build
env:
PR_NUMBER: ${{ github.event.number }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
# Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
TZ: UTC
# - name: Test ${{ github.event.inputs.connector }}
# if: github.event.inputs.run-tests == 'true'
# run: ./tools/integrations/manage.sh test airbyte-integrations/${{ github.event.inputs.connector }}
# - name: Finalize Sentry release
# if: startsWith(github.event.inputs.connector, 'connectors')
# run: |
# sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}"
# env:
# SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
# SENTRY_ORG: airbyte-5j
# SENTRY_PROJECT: airbyte-connectors
# - name: Build and Test Success Comment
# if: github.event.inputs.comment-id && success()
# uses: peter-evans/create-or-update-comment@v1
# with:
# comment-id: ${{ github.event.inputs.comment-id }}
# body: |
# > :rocket: Successfully built and tested ${{github.event.inputs.connector}}
# - name: Build and Test Failure Comment
# if: github.event.inputs.comment-id && !success()
# uses: peter-evans/create-or-update-comment@v1
# with:
# comment-id: ${{ github.event.inputs.comment-id }}
# body: |
# > :x: Failed to build and test ${{github.event.inputs.connector}}
# - name: Slack Notification - Failure
# if: failure()
# uses: rtCamp/action-slack-notify@master
# env:
# SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
# SLACK_USERNAME: Buildozer
# SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
# SLACK_COLOR: DC143C
# SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}"
# SLACK_FOOTER: ""
# - name: Add Final Success Comment
# if: github.event.inputs.comment-id && success()
# uses: peter-evans/create-or-update-comment@v1
# with:
# comment-id: ${{ github.event.inputs.comment-id }}
# body: |
# > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
# - name: Set publish label
# if: success()
# run: |
# echo "set some label on PR"
# In case of self-hosted EC2 errors, remove this block.
stop-bump-build-test-connector-runner:
name: Stop Build EC2 Runner
needs:
- start-bump-build-test-connector-runner # required to get output from the start-runner job
- bump-build-test-connector # required to wait when the main job is done
- find_valid_pat
runs-on: ubuntu-latest
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Stop EC2 runner
uses: supertopher/[email protected]
with:
mode: stop
github-token: ${{ needs.find_valid_pat.outputs.pat }}
label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }}
Loading

0 comments on commit e8406a2

Please sign in to comment.