diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 46a0a12ec79f..06001e2e59d4 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -12,6 +12,12 @@ jobs: - name: Checkout Airbyte uses: actions/checkout@v2 + - name: Launch Integration Tests + if: github.ref == 'refs/heads/master' + run: ./tools/bin/ci_integration_workflow_launcher.sh + env: + GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }} + # - name: Check images exist # run: ./tools/bin/check_images_exist.sh @@ -65,49 +71,6 @@ jobs: if: success() && github.ref == 'refs/heads/master' run: ./tools/site/link_checker.sh check_docs - # this should happen as late as possible so we don't accidentally have tests in the wrong stage depending on creds - - name: Write Integration Test Credentials - run: ./tools/bin/ci_credentials.sh - env: - BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }} - STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }} - GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} - SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} - GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} - GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} - HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} - GSHEETS_INTEGRATION_TESTS_CREDS: ${{ secrets.GSHEETS_INTEGRATION_TESTS_CREDS }} - SNOWFLAKE_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDS }} - ADWORDS_INTEGRATION_TEST_CREDS: ${{ secrets.ADWORDS_INTEGRATION_TEST_CREDS }} - FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }} - SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }} - SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }} - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} - AWS_REDSHIFT_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_REDSHIFT_INTEGRATION_TEST_CREDS }} - MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} - - - name: Run Integration Tests (PR) - if: success() && github.ref != 'refs/heads/master' - run: ./tools/bin/integration_test_pr.sh - - - name: Run Integration Tests (Master) - if: success() && github.ref == 'refs/heads/master' - run: ./gradlew --no-daemon integrationTest --scan - - - name: Run Standard Source Tests (PR) - if: success() && github.ref != 'refs/heads/master' - run: ./tools/bin/standard_test_pr.sh - - # todo (cgardens) - just run pr integration tests, until we can fix the build. - # https://airbytehq.slack.com/archives/C019WEENQRM/p1606012092002600?thread_ts=1605924059.000200&cid=C019WEENQRM - - name: Run Standard Source Tests (Master Tmp) - if: success() && github.ref == 'refs/heads/master' - run: ./tools/bin/standard_test_pr.sh - - # - name: Run Standard Source Tests (Master) - # if: success() && github.ref == 'refs/heads/master' - # run: ./gradlew --no-daemon standardSourceTestPython --scan - - name: Build Core Docker Images if: success() && github.ref == 'refs/heads/master' run: ./gradlew --no-daemon composeBuild --scan diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 22d643daeddc..b96c5d3eb61e 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -16,8 +16,7 @@ jobs: with: token: ${{ secrets.SLASH_COMMAND_PAT }} commands: | - source-exchangeratesapi-singer + test static-args: | ref=${{ steps.getref.outputs.ref }} dispatch-type: workflow - event-type-suffix: "" # used to map this command to integration-test.yml diff --git a/.github/workflows/source-exchangeratesapi-singer.yml b/.github/workflows/source-exchangeratesapi-singer.yml deleted file mode 100644 index 3204583cc1e0..000000000000 --- a/.github/workflows/source-exchangeratesapi-singer.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: source-exchangeratesapi-singer -on: - workflow_dispatch: - schedule: - - cron: '0 */6 * * *' - -jobs: - integration_test: - runs-on: ubuntu-latest - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 - with: - java-version: '14' - - uses: actions/setup-node@v1 - with: - node-version: '14.7' - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - run: | - ./gradlew --no-daemon --scan :airbyte-integrations:connectors:source-exchangeratesapi-singer:integrationTest diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml new file mode 100644 index 000000000000..c8ee880dd758 --- /dev/null +++ b/.github/workflows/test-command.yml @@ -0,0 +1,54 @@ +name: integration-test +on: + workflow_dispatch: + inputs: + connector: + description: 'Airbyte Connector' + required: true + +jobs: + integration_test: + runs-on: ubuntu-latest + steps: + - name: Search for valid integration name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: '^[a-zA-Z0-9-_]+$' + regex_flags: 'i' # required to be set for this plugin + search_string: ${{ github.event.inputs.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != github.event.inputs.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Checkout Airbyte + uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: '14' + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Write Integration Test Credentials + run: ./tools/bin/ci_credentials.sh + env: + BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }} + STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }} + GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} + SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} + GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} + GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} + HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} + GSHEETS_INTEGRATION_TESTS_CREDS: ${{ secrets.GSHEETS_INTEGRATION_TESTS_CREDS }} + SNOWFLAKE_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDS }} + ADWORDS_INTEGRATION_TEST_CREDS: ${{ secrets.ADWORDS_INTEGRATION_TEST_CREDS }} + FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }} + SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }} + SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }} + AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + AWS_REDSHIFT_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_REDSHIFT_INTEGRATION_TEST_CREDS }} + MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} + - run: | + ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle index 0b0e3b25fc54..937fa0185316 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle @@ -14,7 +14,7 @@ dependencies { integrationTestImplementation project(':airbyte-workers') integrationTestImplementation project(':airbyte-config:models') integrationTestImplementation project(':airbyte-protocol:models') - implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } integrationTest.dependsOn airbyteDocker diff --git a/tools/bin/ci_integration_test.sh b/tools/bin/ci_integration_test.sh new file mode 100755 index 000000000000..f5fb539127f0 --- /dev/null +++ b/tools/bin/ci_integration_test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e + +# runs integration and/or standard tests for an integration name + +connector="$1" +all_integration_tests=$(./gradlew integrationTest --dry-run | grep 'integrationTest SKIPPED' | cut -d: -f 4) +all_standard_python_tests=$(./gradlew standardSourceTestPython --dry-run | grep 'standardSourceTestPython SKIPPED' | cut -d: -f 4) + +if [[ "$connector" == "all" ]] ; then + echo "Running: ./gradlew --no-daemon --scan integrationTest standardSourceTestPython" + ./gradlew --no-daemon --scan integrationTest standardSourceTestPython +else + selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector$" || echo "") + selected_standard_python_test=$(echo "$all_standard_python_tests" | grep "^$connector$" || echo "") + integrationTestCommand=":airbyte-integrations:connectors:$connector:integrationTest" + standardPythonTestCommand=":airbyte-integrations:connectors:$connector:standardSourceTestPython" + if [ -n "$selected_integration_test" ] && [ -n "$selected_standard_python_test" ] ; then + echo "Running: ./gradlew --no-daemon --scan $integrationTestCommand $standardPythonTestCommand" + ./gradlew --no-daemon --scan "$integrationTestCommand" "$standardPythonTestCommand" + elif [ -z "$selected_integration_test" ] && [ -n "$selected_standard_python_test" ] ; then + echo "Running: ./gradlew --no-daemon --scan $standardPythonTestCommand" + ./gradlew --no-daemon --scan "$standardPythonTestCommand" + elif [ -n "$selected_integration_test" ] && [ -z "$selected_standard_python_test" ] ; then + echo "Running: ./gradlew --no-daemon --scan $integrationTestCommand" + ./gradlew --no-daemon --scan "$integrationTestCommand" + else + echo "Connector '$connector' not found..." + exit 1 + fi +fi diff --git a/tools/bin/ci_integration_workflow_launcher.sh b/tools/bin/ci_integration_workflow_launcher.sh new file mode 100755 index 000000000000..739bc5b91d89 --- /dev/null +++ b/tools/bin/ci_integration_workflow_launcher.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e + +# launches integration test workflows for master builds + +if [[ -z "$GITHUB_TOKEN" ]] ; then + echo "GITHUB_TOKEN not set..." + exit 1 +fi + +REPO_API=https://api.github.com/repos/airbytehq/airbyte +WORKFLOW_NAME=integration-test +WORKFLOW_ID=$(curl --header "Authorization: Bearer $GITHUB_TOKEN" "$REPO_API/actions/workflows" | jq -r ".workflows[] | select( .name == \"$WORKFLOW_NAME\" ) | .id") +MATCHING_WORKFLOW_IDS=$(wc -l <<< "${WORKFLOW_ID}") + +if [ "$MATCHING_WORKFLOW_IDS" -ne "1" ] ; then + echo "More than one workflow exists with the name $WORKFLOW_NAME" + exit 1 +fi + +MAX_RUNNING_MASTER_WORKFLOWS=5 +RUNNING_MASTER_WORKFLOWS=$(curl "$REPO_API/actions/workflows/$WORKFLOW_ID/runs?branch=master&status=in_progress" --header "Authorization: Bearer $GITHUB_TOKEN" | jq -r ".total_count") +if [ "$RUNNING_MASTER_WORKFLOWS" -gt "$MAX_RUNNING_MASTER_WORKFLOWS" ] ; then + echo "More than $MAX_RUNNING_MASTER_WORKFLOWS integration tests workflows running on master." + echo "Skipping launching workflows." + exit 0 +fi + +CONNECTORS=$(./gradlew integrationTest standardSourceTestPython --dry-run | grep 'integrationTest SKIPPED\|standardSourceTestPython SKIPPED' | cut -d: -f 4) +echo "$CONNECTORS" | while read -r connector ; do + echo "Issuing request for connector $connector..." + curl \ + -X POST \ + --header "Authorization: Bearer $GITHUB_TOKEN" \ + "$REPO_API/actions/workflows/$WORKFLOW_ID/dispatches" \ + -d "{\"ref\":\"master\", \"connector\": \"$connector\"}" +done diff --git a/tools/bin/integration_test_pr.sh b/tools/bin/integration_test_pr.sh deleted file mode 100755 index d44e6e8d699b..000000000000 --- a/tools/bin/integration_test_pr.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -. tools/lib/lib.sh - -assert_root - -./gradlew --no-daemon --scan \ - :airbyte-integrations:connectors:destination-bigquery:integrationTest \ - :airbyte-integrations:connectors:destination-postgres:integrationTest \ - :airbyte-integrations:connectors:destination-csv:integrationTest \ - :airbyte-integrations:connectors:source-postgres:integrationTest \ - :airbyte-integrations:connectors:source-stripe-singer:integrationTest \ - :airbyte-integrations:connectors:source-exchangeratesapi-singer:integrationTest diff --git a/tools/bin/standard_test_pr.sh b/tools/bin/standard_test_pr.sh deleted file mode 100755 index a1d80a813f56..000000000000 --- a/tools/bin/standard_test_pr.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -e - -. tools/lib/lib.sh - -assert_root - -./gradlew --no-daemon --scan \ - :airbyte-integrations:connectors:source-github-singer:standardSourceTestPython \ - :airbyte-integrations:connectors:source-marketo-singer:standardSourceTestPython