diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index a9bfb1c7e5e0..3b4cd245a375 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -66,16 +66,28 @@ jobs: uses: actions/checkout@v2 with: repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository - # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559. - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - uses: actions/setup-java@v1 + - name: Install Java + uses: actions/setup-java@v1 with: java-version: '17' - - name: Write Integration Test Credentials # TODO DRY this with test-command.yml - run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }} + - name: Install Pyenv and Tox + # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/ + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ github.event.inputs.connector }} env: - GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }} GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index e445470f975a..000c6dd993b0 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -61,20 +61,33 @@ jobs: uses: actions/checkout@v2 with: repository: ${{ github.event.inputs.repo }} - # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/ - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - uses: actions/setup-java@v1 + - name: Install Java + uses: actions/setup-java@v1 with: java-version: '17' - - name: Write Integration Test Credentials - run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }} + - name: Install Pyenv and Tox + # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/ + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ github.event.inputs.connector }} env: - GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }} GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - run: | + + - name: test ${{ github.event.inputs.connector }} + run: | ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} - name: test ${{ github.event.inputs.connector }} id: test env: ACTION_RUN_ID: ${{github.run_id}} diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index 8413859420cb..3db5a501d246 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -67,13 +67,24 @@ jobs: uses: actions/checkout@v2 with: repository: ${{ github.event.inputs.repo }} - # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/ - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - name: Write Integration Test Credentials - run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }} + - name: Install Pyenv and Tox + # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/ + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ github.event.inputs.connector }} env: - GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }} GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: | ./tools/bin/ci_performance_test.sh ${{ github.event.inputs.connector }} ${{ github.event.inputs.cpulimit }} ${{ github.event.inputs.memorylimit }} diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index dedfe541ed03..a4b6f270e081 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -138,7 +138,8 @@ In order to run integration tests in CI, you'll often need to inject credentials 2. **Add the GSM secret's labels**: * `connector` (required) -- unique connector's name or set of connectors' names with '_' as delimiter i.e.: `connector=source-s3`, `connector=destination-snowflake` * `filename` (optional) -- custom target secret file. Unfortunately Google doesn't use '.' into labels' values and so Airbyte CI scripts will add '.json' to the end automatically. By default secrets will be saved to `./secrets/config.json` i.e: `filename=config_auth` => `secrets/config_auth.json` -3. That should be it. +3. **Save a necessary JSON value** [Example](https://user-images.githubusercontent.com/11213273/146040653-4a76c371-a00e-41fe-8300-cbd411f10b2e.png). +4. That should be it. #### Access CI secrets on GSM Access to GSM storage is limited to Airbyte employees. To give an employee permissions to the project: @@ -148,9 +149,4 @@ Access to GSM storage is limited to Airbyte employees. To give an employee permi - select the role `Development_CI_Secrets` 3. Save -#### How to migrate to the new secrets' logic: -1. Create all necessary secrets according to the instructions above -2. Remove all lines with old connector's Github secrets from this file: tools/bin/ci_credentials.sh -3. Remove all old secrets from Github repository secrets. -4. That should be it. diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh deleted file mode 100755 index bdf0e35d3b9e..000000000000 --- a/tools/bin/ci_credentials.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash - -. tools/lib/lib.sh -. tools/lib/gcp-token.sh - -set -e - -# all secrets will be loaded if the second argument is not present -CONNECTOR_FULLNAME=${1:-all} -CONNECTOR_NAME=`echo ${CONNECTOR_FULLNAME} | rev | cut -d'/' -f1 | rev` - -GSM_SCOPES="https://www.googleapis.com/auth/cloud-platform" - -# If a secret is available in both Github and GSM, then the GSM secret is used, otherwise Github. - -declare -A SECRET_MAP - - -function read_secrets() { - local connector_name=$1 - local creds=$2 - local cred_filename=${3:-config.json} - local secrets_provider_name=${4:-github} - - [ -z "$connector_name" ] && error "Empty connector name" - [ -z "$creds" ] && echo "!!!!!Creds not set for the connector $connector_name from ${secrets_provider_name}" - - if [[ $CONNECTOR_NAME != "all" && ${connector_name} != ${CONNECTOR_NAME} ]]; then - return 0 - fi - local key="${connector_name}#${cred_filename}" - [[ -z "${creds}" ]] && error "Empty credential for the connector '${key} from ${secrets_provider_name}" - - if [ -v SECRET_MAP[${key}] ]; then - echo "The connector '${key}' was added before" - return 0 - fi - echo "register the secret ${key} from ${secrets_provider_name}" - SECRET_MAP[${key}]="${creds}" - return 0 -} - -function write_secret_to_disk() { - local connector_name=$1 - local cred_filename=$2 - local creds=$3 - if jq -e . >/dev/null 2>&1 <<< "${creds}"; then - echo "Parsed JSON for '${connector_name}' => ${cred_filename} successfully" - else - error "Failed to parse JSON for '${connector_name}' => ${cred_filename}" - fi - - if [ "$connector_name" = "base-normalization" ]; then - local secrets_dir="airbyte-integrations/bases/${connector_name}/secrets" - else - local secrets_dir="airbyte-integrations/connectors/${connector_name}/secrets" - fi - mkdir -p "$secrets_dir" - echo "Saved a secret => ${secrets_dir}/${cred_filename}" - echo "$creds" > "${secrets_dir}/${cred_filename}" -} - -function write_all_secrets() { - for key in "${!SECRET_MAP[@]}"; do - local connector_name=$(echo ${key} | cut -d'#' -f1) - local cred_filename=$(echo ${key} | cut -d'#' -f2) - local creds=${SECRET_MAP[${key}]} - write_secret_to_disk ${connector_name} ${cred_filename} "${creds}" - - done - return 0 -} - - -function export_github_secrets(){ - # We expect that all secrets injected from github are available in an env variable `SECRETS_JSON` - local pairs=`echo ${GITHUB_PROVIDED_SECRETS_JSON} | jq -c 'keys[] as $k | {"name": $k, "value": .[$k]} | @base64'` - while read row; do - pair=$(echo "${row}" | tr -d '"' | base64 -d) - local key=$(echo ${pair} | jq -r .name) - local value=$(echo ${pair} | jq -r .value) - if [[ "$key" == *"_CREDS"* ]]; then - declare -gxr "${key}"="$(echo ${value})" - fi - done <<< ${pairs} - unset GITHUB_PROVIDED_SECRETS_JSON -} - -function export_gsm_secrets(){ - local config_file=`mktemp` - echo "${GCP_GSM_CREDENTIALS}" > ${config_file} - local access_token=$(get_gcp_access_token "${config_file}" "${GSM_SCOPES}") - local project_id=$(parse_project_id "${config_file}") - rm ${config_file} - - # docs: https://cloud.google.com/secret-manager/docs/filtering#api - local filter="name:SECRET_" - [[ ${CONNECTOR_NAME} != "all" ]] && filter="${filter} AND labels.connector=${CONNECTOR_NAME}" - local uri="https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets" - local next_token='' - while true; do - local data=$(curl -s --get --fail "${uri}" \ - --data-urlencode "filter=${filter}" \ - --data-urlencode "pageToken=${next_token}" \ - --header "authorization: Bearer ${access_token}" \ - --header "content-type: application/json" \ - --header "x-goog-user-project: ${project_id}") - [[ -z ${data} ]] && error "Can't load secret for connector ${CONNECTOR_NAME}" - # GSM returns an empty JSON object if secrets are not found. - # It breaks JSON parsing by the 'jq' utility. The simplest fix is response normalization - [[ ${data} == "{}" ]] && data='{"secrets": []}' - - for row in $(echo "${data}" | jq -r '.secrets[] | @base64'); do - local secret_info=$(echo ${row} | base64 --decode) - local secret_name=$(echo ${secret_info}| jq -r .name) - local label_filename=$(echo ${secret_info}| jq -r '.labels.filename // "config"') - local label_connectors=$(echo ${secret_info}| jq -r '.labels.connector // ""') - - # skip secrets without the label "connector" - [[ -z ${label_connectors} ]] && continue - if [[ "$label_connectors" != *"${CONNECTOR_NAME}"* ]]; then - echo "Not found ${CONNECTOR_NAME} info into the label 'connector' of the secret ${secret_name}" - continue - fi - - # all secret file names should be finished with ".json" - # but '.' cant be used in google, so we append it - local filename="${label_filename}.json" - echo "found the Google secret of ${label_connectors}: ${secret_name} => ${filename}" - local secret_uri="https://secretmanager.googleapis.com/v1/${secret_name}/versions/latest:access" - local secret_data=$(curl -s --get --fail "${secret_uri}" \ - --header "authorization: Bearer ${access_token}" \ - --header "content-type: application/json" \ - --header "x-goog-user-project: ${project_id}") - [[ -z ${secret_data} ]] && error "Can't load secrets' list" - - secret_data=$(echo ${secret_data} | jq -r '.payload.data // ""' | base64 -d) - read_secrets "${CONNECTOR_NAME}" "${secret_data}" "${filename}" "gsm" - done - next_token=`echo ${data} | jq -r '.nextPageToken // ""'` - [[ -z ${next_token} ]] && break - done - return 0 -} - -export_gsm_secrets -export_github_secrets - - - -# Please maintain this organisation and alphabetise. -read_secrets destination-amazon-sqs "$DESTINATION_AMAZON_SQS_CREDS" -read_secrets destination-bigquery "$BIGQUERY_INTEGRATION_TEST_CREDS" "credentials.json" -read_secrets destination-bigquery-denormalized "$BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS" "credentials.json" -read_secrets destination-databricks "$DESTINATION_DATABRICKS_CREDS" -read_secrets destination-firestore "$DESTINATION_FIRESTORE_CREDS" -read_secrets destination-gcs "$DESTINATION_GCS_CREDS" -read_secrets destination-kvdb "$DESTINATION_KVDB_TEST_CREDS" -read_secrets destination-keen "$DESTINATION_KEEN_TEST_CREDS" - -read_secrets destination-postgres "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json" -read_secrets destination-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" -read_secrets destination-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json" -read_secrets destination-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" -read_secrets destination-pubsub "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json" -read_secrets destination-rabbitmq "$DESTINATION_RABBITMQ_TEST_CREDS" -read_secrets destination-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" -read_secrets destination-dynamodb "$DESTINATION_DYNAMODB_TEST_CREDS" -read_secrets destination-oracle "$AWS_ORACLE_INTEGRATION_TEST_CREDS" -read_secrets destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS" -read_secrets destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS" -read_secrets destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json" -read_secrets destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json" -read_secrets destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json" - -read_secrets base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json" -read_secrets base-normalization "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "snowflake.json" -read_secrets base-normalization "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" "redshift.json" -read_secrets base-normalization "$AWS_ORACLE_INTEGRATION_TEST_CREDS" "oracle.json" - -read_secrets source-airtable "$SOURCE_AIRTABLE_TEST_CREDS" -read_secrets source-amazon-seller-partner "$AMAZON_SELLER_PARTNER_TEST_CREDS" -read_secrets source-amazon-sqs "$SOURCE_AMAZON_SQS_TEST_CREDS" -read_secrets source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS" -read_secrets source-apify-dataset "$APIFY_INTEGRATION_TEST_CREDS" -read_secrets source-amazon-ads "$AMAZON_ADS_TEST_CREDS" -read_secrets source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS" -read_secrets source-asana "$SOURCE_ASANA_TEST_CREDS" -read_secrets source-aws-cloudtrail "$SOURCE_AWS_CLOUDTRAIL_CREDS" -read_secrets source-azure-table "$SOURCE_AZURE_TABLE_CREDS" -read_secrets source-bamboo-hr "$SOURCE_BAMBOO_HR_CREDS" -read_secrets source-bigcommerce "$SOURCE_BIGCOMMERCE_CREDS" -read_secrets source-bigquery "$BIGQUERY_TEST_CREDS" "credentials.json" -read_secrets source-bing-ads "$SOURCE_BING_ADS_CREDS" -read_secrets source-braintree "$BRAINTREE_TEST_CREDS" -read_secrets source-cart "$CART_TEST_CREDS" -read_secrets source-chargebee "$CHARGEBEE_INTEGRATION_TEST_CREDS" -read_secrets source-close-com "$SOURCE_CLOSE_COM_CREDS" -read_secrets source-commercetools "$SOURCE_COMMERCETOOLS_TEST_CREDS" -read_secrets source-confluence "$SOURCE_CONFLUENCE_TEST_CREDS" -read_secrets source-delighted "$SOURCE_DELIGHTED_TEST_CREDS" -read_secrets source-drift "$DRIFT_INTEGRATION_TEST_CREDS" -read_secrets source-drift "$DRIFT_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json" -read_secrets source-dixa "$SOURCE_DIXA_TEST_CREDS" -read_secrets source-exchange-rates "$EXCHANGE_RATES_TEST_CREDS" -read_secrets source-file "$GOOGLE_CLOUD_STORAGE_TEST_CREDS" "gcs.json" -read_secrets source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json" -read_secrets source-file "$AZURE_STORAGE_INTEGRATION_TEST_CREDS" "azblob.json" -read_secrets source-file "$FILE_SECURE_HTTPS_TEST_CREDS" -read_secrets source-file-secure "$FILE_SECURE_HTTPS_TEST_CREDS" -read_secrets source-freshdesk "$FRESHDESK_TEST_CREDS" -read_secrets source-freshsales "$SOURCE_FRESHSALES_TEST_CREDS" -read_secrets source-freshservice "$SOURCE_FRESHSERVICE_TEST_CREDS" -read_secrets source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" -read_secrets source-facebook-pages "$FACEBOOK_PAGES_INTEGRATION_TEST_CREDS" -read_secrets source-gitlab "$GITLAB_INTEGRATION_TEST_CREDS" -read_secrets source-github "$GH_NATIVE_INTEGRATION_TEST_CREDS" -read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS" -read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json" -read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json" -read_secrets source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS" -read_secrets source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json" -read_secrets source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS" -read_secrets source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json" -read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS" -read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS" -read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" -read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" -read_secrets source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS" -read_secrets source-intercom "$INTERCOM_INTEGRATION_OAUTH_TEST_CREDS" "config_apikey.json" -read_secrets source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS" -read_secrets source-jira "$JIRA_INTEGRATION_TEST_CREDS" -read_secrets source-klaviyo "$KLAVIYO_TEST_CREDS" -read_secrets source-lemlist "$SOURCE_LEMLIST_TEST_CREDS" -read_secrets source-lever-hiring "$LEVER_HIRING_INTEGRATION_TEST_CREDS" -read_secrets source-looker "$LOOKER_INTEGRATION_TEST_CREDS" -read_secrets source-linnworks "$SOURCE_LINNWORKS_TEST_CREDS" -read_secrets source-mailchimp "$MAILCHIMP_TEST_CREDS" -read_secrets source-marketo "$SOURCE_MARKETO_TEST_CREDS" -read_secrets source-mixpanel "$MIXPANEL_INTEGRATION_TEST_CREDS" -read_secrets source-monday "$SOURCE_MONDAY_TEST_CREDS" -read_secrets source-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" -read_secrets source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json" -read_secrets source-mssql "$MSSQL_RDS_TEST_CREDS" -read_secrets source-notion "$SOURCE_NOTION_TEST_CREDS" -read_secrets source-okta "$SOURCE_OKTA_TEST_CREDS" -read_secrets source-onesignal "$SOURCE_ONESIGNAL_TEST_CREDS" -read_secrets source-outreach "$SOURCE_OUTREACH_TEST_CREDS" -read_secrets source-plaid "$PLAID_INTEGRATION_TEST_CREDS" -read_secrets source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS" -read_secrets source-pinterest "$PINTEREST_TEST_CREDS" -read_secrets source-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json" -read_secrets source-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" -read_secrets source-posthog "$POSTHOG_TEST_CREDS" -read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS" "config.json" -read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH" "oauth_config.json" -read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD" "old_config.json" -read_secrets source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS" -read_secrets source-recharge "$RECHARGE_INTEGRATION_TEST_CREDS" -read_secrets source-recurly "$SOURCE_RECURLY_INTEGRATION_TEST_CREDS" -read_secrets source-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" -read_secrets source-s3 "$SOURCE_S3_TEST_CREDS" -read_secrets source-s3 "$SOURCE_S3_PARQUET_CREDS" "parquet_config.json" -read_secrets source-salesloft "$SOURCE_SALESLOFT_TEST_CREDS" -read_secrets source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS" -read_secrets source-shopify "$SHOPIFY_INTEGRATION_TEST_CREDS" -read_secrets source-shopify "$SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json" -read_secrets source-shortio "$SOURCE_SHORTIO_TEST_CREDS" -read_secrets source-slack "$SOURCE_SLACK_TEST_CREDS" -read_secrets source-slack "$SOURCE_SLACK_OAUTH_TEST_CREDS" "config_oauth.json" -read_secrets source-smartsheets "$SMARTSHEETS_TEST_CREDS" -read_secrets source-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "config.json" -read_secrets source-square "$SOURCE_SQUARE_CREDS" -read_secrets source-strava "$SOURCE_STRAVA_TEST_CREDS" -read_secrets source-paystack "$SOURCE_PAYSTACK_TEST_CREDS" -read_secrets source-sentry "$SOURCE_SENTRY_TEST_CREDS" -read_secrets source-stripe "$SOURCE_STRIPE_CREDS" -read_secrets source-stripe "$STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS" "connected_account_config.json" -read_secrets source-surveymonkey "$SURVEYMONKEY_TEST_CREDS" -read_secrets source-tempo "$TEMPO_INTEGRATION_TEST_CREDS" -read_secrets source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_TEST_CREDS" -read_secrets source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS" "prod_config.json" -read_secrets source-twilio "$TWILIO_TEST_CREDS" -read_secrets source-typeform "$SOURCE_TYPEFORM_CREDS" -read_secrets source-us-census "$SOURCE_US_CENSUS_TEST_CREDS" -read_secrets source-zendesk-chat "$ZENDESK_CHAT_INTEGRATION_TEST_CREDS" -read_secrets source-zendesk-sunshine "$ZENDESK_SUNSHINE_TEST_CREDS" -read_secrets source-zendesk-support "$ZENDESK_SUPPORT_TEST_CREDS" -read_secrets source-zendesk-support "$ZENDESK_SUPPORT_OAUTH_TEST_CREDS" "config_oauth.json" -read_secrets source-zendesk-talk "$ZENDESK_TALK_TEST_CREDS" -read_secrets source-zenloop "$SOURCE_ZENLOOP_TEST_CREDS" -read_secrets source-zoom-singer "$ZOOM_INTEGRATION_TEST_CREDS" -read_secrets source-zuora "$SOURCE_ZUORA_TEST_CREDS" - -write_all_secrets -exit $? - diff --git a/tools/ci_common_utils/ci_common_utils/__init__.py b/tools/ci_common_utils/ci_common_utils/__init__.py new file mode 100644 index 000000000000..0078efdb784a --- /dev/null +++ b/tools/ci_common_utils/ci_common_utils/__init__.py @@ -0,0 +1,7 @@ +from .google_api import GoogleApi +from .logger import Logger + +__all__ = ( + "Logger", + "GoogleApi", +) diff --git a/tools/ci_common_utils/ci_common_utils/google_api.py b/tools/ci_common_utils/ci_common_utils/google_api.py new file mode 100644 index 000000000000..2ce4ef2a0f7c --- /dev/null +++ b/tools/ci_common_utils/ci_common_utils/google_api.py @@ -0,0 +1,88 @@ +import time +from dataclasses import dataclass +from typing import Mapping, Any, List, ClassVar + +import jwt +import requests + +from .logger import Logger + +TOKEN_TTL = 3600 + + +@dataclass +class GoogleApi: + """ + Simple Google API client + """ + logger: ClassVar[Logger] = Logger() + + config: Mapping[str, Any] + scopes: List[str] + _access_token: str = None + + def get(self, url: str, params: Mapping = None) -> Mapping[str, Any]: + """Sends a GET request""" + token = self.get_access_token() + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "X-Goog-User-Project": self.project_id + } + # Making a get request + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + return response.json() + + def post(self, url: str, json: Mapping = None, params: Mapping = None) -> Mapping[str, Any]: + """Sends a POST request""" + token = self.get_access_token() + + headers = { + "Authorization": f"Bearer {token}", + "X-Goog-User-Project": self.project_id + } + # Making a get request + response = requests.post(url, headers=headers, json=json, params=params) + try: + response.raise_for_status() + except Exception: + self.logger.error(f"error body: {response.text}") + raise + return response.json() + + @property + def token_uri(self): + return self.config["token_uri"] + + @property + def project_id(self): + return self.config["project_id"] + + def __generate_jwt(self) -> str: + """Generates JWT token by a service account json file and scopes""" + now = int(time.time()) + claim = { + "iat": now, + "iss": self.config["client_email"], + "scope": ",".join(self.scopes), + "aud": self.token_uri, + "exp": now + TOKEN_TTL, + } + return jwt.encode(claim, self.config["private_key"].encode(), algorithm="RS256") + + def get_access_token(self) -> str: + """Generates an access token by a service account json file and scopes""" + + if self._access_token is None: + self._access_token = self.__get_access_token() + + return self._access_token + + def __get_access_token(self) -> str: + jwt = self.__generate_jwt() + resp = requests.post(self.token_uri, data={ + "assertion": jwt, + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + }) + return resp.json()["access_token"] diff --git a/tools/ci_common_utils/ci_common_utils/logger.py b/tools/ci_common_utils/ci_common_utils/logger.py new file mode 100644 index 000000000000..00ec0a7c7162 --- /dev/null +++ b/tools/ci_common_utils/ci_common_utils/logger.py @@ -0,0 +1,81 @@ +import datetime as dt +import inspect +import logging +import logging.handlers +import sys +from typing import Callable + + +class MyFormatter(logging.Formatter): + """Custom formatter for logging + """ + converter = dt.datetime.fromtimestamp + + def formatTime(self, record, datefmt=None): + """! @brief redefinition of format of log + """ + ct = self.converter(record.created) + if datefmt: + s = ct.strftime(datefmt) + else: + t = ct.strftime("%Y-%m-%d %H:%M:%S") + s = "%s,%03d" % (t, record.msecs) + return s + + +class Logger: + """Simple logger with a pretty log header + the method error returns the value 1 + the method critical terminates a script work + """ + + def __init__(self): + formatter = MyFormatter( + fmt='[%(asctime)s] - %(levelname)-6s - %(message)s', + datefmt='%d/%m/%Y %H:%M:%S.%f') + + logger_name = __name__ + stack_items = inspect.stack() + for i in range(len(stack_items)): + if stack_items[i].filename.endswith("ci_common_utils/logger.py"): + logger_name = ".".join(stack_items[i + 1].filename.split("/")[-3:])[:-3] + + self._logger = logging.getLogger(logger_name) + self._logger.setLevel(logging.DEBUG) + self._logger.propagate = False + + handler = logging.StreamHandler() + handler.setLevel(logging.DEBUG) + handler.setFormatter(formatter) + self._logger.addHandler(handler) + + @classmethod + def __prepare_log_line(cls, func_name: str, func: Callable) -> Callable: + def wrapper(*args): + prefix = "" + stack_items = inspect.stack() + for i in range(len(stack_items)): + if stack_items[i].filename.endswith("ci_common_utils/logger.py"): + filepath = stack_items[i + 1].filename + line_number = stack_items[i + 1].lineno + + # show last 3 path items only + filepath = "/".join(filepath.split("/")[-3:]) + prefix = f"[{filepath}:{line_number}]" + break + if prefix: + args = list(args) + args[0] = f"{prefix} # {args[0]}" + func(*args) + if func_name == "critical": + sys.exit(1) + elif func_name == "error": + return 1 + return 0 + + return wrapper + + def __getattr__(self, function_name: str): + if not hasattr(self._logger, function_name): + return super().__getattr__(function_name) + return self.__prepare_log_line(function_name, getattr(self._logger, function_name), ) diff --git a/tools/ci_common_utils/setup.py b/tools/ci_common_utils/setup.py new file mode 100644 index 000000000000..8e24d60c2909 --- /dev/null +++ b/tools/ci_common_utils/setup.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["cryptography", "requests", "pyjwt~=2.3.0"] + +TEST_REQUIREMENTS = ["requests-mock"] + +setup( + version="0.0.0", + name="ci_common_utils", + description="Suite of all often used classes and common functions", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + python_requires='>=3.7', + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/tools/ci_common_utils/tests/__init__.py b/tools/ci_common_utils/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/ci_common_utils/tests/test_logger.py b/tools/ci_common_utils/tests/test_logger.py new file mode 100644 index 000000000000..e45464550edf --- /dev/null +++ b/tools/ci_common_utils/tests/test_logger.py @@ -0,0 +1,45 @@ +import re +from datetime import datetime, timedelta + +import pytest + +from ci_common_utils import Logger + +LOG_RE = re.compile( + r'^\[(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\.\d{6})\] -' + r'\s+(\w+)\s+- \[.*tests/test_logger.py:(\d+)\] # (.+)') +LOGGER = Logger() +TEST_MESSAGE = 'sbhY=)9\'v-}LT=)jjF66(XrZh=]>7Xp"?/zCz,=eu8K47u8' + + +def check_output(msg: str, expected_line_number: int, expected_log_level: str): + m = LOG_RE.match(msg) + assert m is not None, f"incorrect message format, pattern: {LOG_RE.pattern}" + date_time, log_level, line_number, msg = m.groups() + + assert int(line_number) == expected_line_number + assert expected_log_level == log_level + assert expected_log_level == log_level + dt = datetime.strptime(date_time, '%d/%m/%Y %H:%M:%S.%f') + now = datetime.now() + delta = timedelta(seconds=1) + assert now - delta < dt < now + + +@pytest.mark.parametrize('log_func,expected_log_level,expected_code', ( + (LOGGER.debug, 'DEBUG', 0), + (LOGGER.warning, 'WARNING', 0), + (LOGGER.info, 'INFO', 0), + (LOGGER.error, 'ERROR', 1) +)) +def test_log_message(capfd, log_func, expected_log_level, expected_code): + assert log_func(TEST_MESSAGE) == expected_code + _, err = capfd.readouterr() + check_output(err, 36, expected_log_level) + + +def test_critical_message(capfd): + with pytest.raises(SystemExit) as (err): + LOGGER.critical(TEST_MESSAGE) + _, err = capfd.readouterr() + check_output(err, 43, 'CRITICAL') diff --git a/tools/ci_credentials/ci_credentials/__init__.py b/tools/ci_credentials/ci_credentials/__init__.py new file mode 100644 index 000000000000..4f55c3781545 --- /dev/null +++ b/tools/ci_credentials/ci_credentials/__init__.py @@ -0,0 +1,6 @@ +# from .main import main +from .secrets_loader import SecretsLoader + +__all__ = ( + "SecretsLoader", +) diff --git a/tools/ci_credentials/ci_credentials/main.py b/tools/ci_credentials/ci_credentials/main.py new file mode 100644 index 000000000000..3147509e3711 --- /dev/null +++ b/tools/ci_credentials/ci_credentials/main.py @@ -0,0 +1,43 @@ +import json +import os +import sys +from json.decoder import JSONDecodeError + +from ci_common_utils import Logger +from . import SecretsLoader + +logger = Logger() + +ENV_GCP_GSM_CREDENTIALS = "GCP_GSM_CREDENTIALS" + + +# credentials of GSM and GitHub secrets should be shared via shell environment + +def main() -> int: + if len(sys.argv) != 2: + return logger.error("uses one script argument only: ") + + # parse unique connector name, because it can have the common prefix "connectors/" + connector_name = sys.argv[1].split("/")[-1] + if connector_name == "all": + # if needed to load all secrets + connector_name = None + + # parse GCP_GSM_CREDENTIALS + try: + gsm_credentials = json.loads(os.getenv(ENV_GCP_GSM_CREDENTIALS) or "{}") + except JSONDecodeError as e: + return logger.error(f"incorrect GCP_GSM_CREDENTIALS value, error: {e}") + + if not gsm_credentials: + return logger.error("GCP_GSM_CREDENTIALS shouldn't be empty!") + + loader = SecretsLoader( + connector_name=connector_name, + gsm_credentials=gsm_credentials, + ) + return loader.write_to_storage(loader.read_from_gsm()) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/ci_credentials/ci_credentials/secrets_loader.py b/tools/ci_credentials/ci_credentials/secrets_loader.py new file mode 100644 index 000000000000..931338cd9ff9 --- /dev/null +++ b/tools/ci_credentials/ci_credentials/secrets_loader.py @@ -0,0 +1,167 @@ +import base64 +import json +from json.decoder import JSONDecodeError +from typing import Mapping, Any, Tuple, ClassVar +from pathlib import Path +from ci_common_utils import GoogleApi +from ci_common_utils import Logger + +DEFAULT_SECRET_FILE = "config" +DEFAULT_SECRET_FILE_WITH_EXT = DEFAULT_SECRET_FILE + ".json" + +GSM_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) + + +class SecretsLoader: + """Loading and saving all requested secrets into connector folders""" + + logger: ClassVar[Logger] = Logger() + base_folder = Path("/actions-runner/_work/airbyte/airbyte") + + def __init__(self, connector_name: str, gsm_credentials: Mapping[str, Any]): + self.gsm_credentials = gsm_credentials + self.connector_name = connector_name + self._api = None + + @property + def api(self) -> GoogleApi: + if self._api is None: + self._api = GoogleApi(self.gsm_credentials, GSM_SCOPES) + return self._api + + def __load_gsm_secrets(self) -> Mapping[Tuple[str, str], str]: + """Loads needed GSM secrets""" + secrets = {} + # docs: https://cloud.google.com/secret-manager/docs/filtering#api + filter = "name:SECRET_" + if self.connector_name: + filter += f" AND labels.connector={self.connector_name}" + url = f"https://secretmanager.googleapis.com/v1/projects/{self.api.project_id}/secrets" + next_token = None + while True: + params = { + "filter": filter, + } + if next_token: + params["pageToken"] = next_token + + data = self.api.get(url, params=params) + for secret_info in data.get("secrets") or []: + secret_name = secret_info["name"] + connector_name = secret_info.get("labels", {}).get("connector") + if not connector_name: + self.logger.warning(f"secret {secret_name} doesn't have the label 'connector'") + continue + elif self.connector_name and connector_name != self.connector_name: + self.logger.warning(f"incorrect the label connector '{connector_name}' of secret {secret_name}") + continue + filename = secret_info.get("labels", {}).get("filename") + if filename: + # all secret file names should be finished with ".json" + # but '.' cant be used in google, so we append it + filename = f"{filename}.json" + else: + # the "filename" label is optional. + filename = DEFAULT_SECRET_FILE_WITH_EXT + log_name = f'{secret_name.split("/")[-1]}({connector_name})' + self.logger.info(f"found GSM secret: {log_name} = > {filename}") + secret_url = f"https://secretmanager.googleapis.com/v1/{secret_name}/versions/latest:access" + + data = self.api.get(secret_url) + secret_value = data.get("payload", {}).get("data") + if not secret_value: + self.logger.warning(f"{log_name} has empty value") + continue + secret_value = base64.b64decode(secret_value.encode()).decode('utf-8') + try: + # minimize and validate its JSON value + secret_value = json.dumps(json.loads(secret_value), separators=(',', ':')) + except JSONDecodeError as err: + self.logger.error(f"{log_name} has non-JSON value!!! Error: {err}") + continue + secrets[(connector_name, filename)] = secret_value + next_token = data.get("nextPageToken") + if not next_token: + break + return secrets + + @staticmethod + def generate_secret_name(connector_name: str, filename: str) -> str: + """ + Generates an unique GSM secret name. + Format of secret name: SECRET____CREDS + Examples: + 1. connector_name: source-linnworks, filename: dsdssds_a-b---_---_config.json + => SECRET_SOURCE-LINNWORKS_DSDSSDS_A-B__CREDS + 2. connector_name: source-s3, filename: config.json + => SECRET_SOURCE-LINNWORKS__CREDS + """ + name_parts = ["secret", connector_name] + filename_wo_ext = filename.replace(".json", "") + if filename_wo_ext != DEFAULT_SECRET_FILE: + name_parts.append(filename_wo_ext.replace(DEFAULT_SECRET_FILE, "").strip("_-")) + name_parts.append("_creds") + return "_".join(name_parts).upper() + + def create_secret(self, connector_name: str, filename: str, secret_value: str) -> bool: + """ + Creates a new GSM secret with auto-generated name. + """ + secret_name = self.generate_secret_name(connector_name, filename) + self.logger.info(f"Generated the new secret name '{secret_name}' for {connector_name}({filename})") + params = { + "secretId": secret_name, + } + labels = { + "connector": connector_name, + } + if filename != DEFAULT_SECRET_FILE: + labels["filename"] = filename.replace(".json", "") + body = { + "labels": labels, + "replication": {"automatic": {}}, + } + url = f"https://secretmanager.googleapis.com/v1/projects/{self.api.project_id}/secrets" + data = self.api.post(url, json=body, params=params) + + # try to create a new version + secret_name = data["name"] + self.logger.info(f"the GSM secret {secret_name} was created") + secret_url = f'https://secretmanager.googleapis.com/v1/{secret_name}:addVersion' + body = { + "payload": {"data": base64.b64encode(secret_value.encode()).decode("utf-8")} + } + self.api.post(secret_url, json=body) + return True + + def read_from_gsm(self) -> int: + """Reads all necessary secrets from different sources""" + secrets = self.__load_gsm_secrets() + + for k in secrets: + if not isinstance(secrets[k], tuple): + secrets[k] = ("GSM", secrets[k]) + source, _ = secrets[k] + self.logger.info(f"Register the file {k[1]}({k[0]}) from {source}") + + if not len(secrets): + return self.logger.critical(f"not found any secrets of the connector '{self.connector_name}'") + return {k: v[1] for k, v in secrets.items()} + + def write_to_storage(self, secrets: Mapping[Tuple[str, str], str]) -> int: + """Tries to save target secrets to the airbyte-integrations/connectors|bases/{connector_name}/secrets folder""" + if not secrets: + return 1 + for (connector_name, filename), secret_value in secrets.items(): + if connector_name == "base-normalization": + secrets_dir = f"airbyte-integrations/bases/{connector_name}/secrets" + else: + secrets_dir = f"airbyte-integrations/connectors/{connector_name}/secrets" + + secrets_dir = self.base_folder / secrets_dir + secrets_dir.mkdir(parents=True, exist_ok=True) + filepath = secrets_dir / filename + with open(filepath, "w") as file: + file.write(secret_value) + self.logger.info(f"The file {filepath} was saved") + return 0 diff --git a/tools/ci_credentials/setup.py b/tools/ci_credentials/setup.py new file mode 100644 index 000000000000..6c072e755223 --- /dev/null +++ b/tools/ci_credentials/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["requests", "ci_common_utils", "pytest"] + +TEST_REQUIREMENTS = ["requests-mock"] + +setup( + version="0.0.0", + name="ci_credentials", + description="Load and extract CI secrets for test suites", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + python_requires='>=3.7', + extras_require={ + "tests": TEST_REQUIREMENTS, + }, + entry_points={ + 'console_scripts': [ + 'ci_credentials = ci_credentials.main:main', + ], + }, +) diff --git a/tools/ci_credentials/tests/__init__.py b/tools/ci_credentials/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/ci_credentials/tests/test_secrets.py b/tools/ci_credentials/tests/test_secrets.py new file mode 100644 index 000000000000..627aadb0487f --- /dev/null +++ b/tools/ci_credentials/tests/test_secrets.py @@ -0,0 +1,158 @@ +import base64 +import json +import re +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +import requests_mock +from ci_credentials.main import ENV_GCP_GSM_CREDENTIALS +from ci_credentials.main import main + +from ci_credentials import SecretsLoader + +HERE = Path(__file__).resolve().parent +TEST_CONNECTOR_NAME = "source-test" +TEMP_FOLDER = Path(tempfile.mkdtemp()) + + +@pytest.fixture(autouse=True, scope="session") +def temp_folder(): + yield + shutil.rmtree(TEMP_FOLDER) + + +@pytest.mark.parametrize( + "connector_name,filename,expected_name", + ( + ("source-default", "config.json", "SECRET_SOURCE-DEFAULT__CREDS"), + ("source-custom-filename-1", "config_custom.json", "SECRET_SOURCE-CUSTOM-FILENAME-1_CUSTOM__CREDS"), + ("source-custom-filename-2", "auth.json", "SECRET_SOURCE-CUSTOM-FILENAME-2_AUTH__CREDS"), + ("source-custom-filename-3", "config_auth-test---___---config.json", + "SECRET_SOURCE-CUSTOM-FILENAME-3_AUTH-TEST__CREDS"), + ("source-custom-filename-4", "_____config_test---config.json", + "SECRET_SOURCE-CUSTOM-FILENAME-4_TEST__CREDS"), + ) +) +def test_secret_name_generation(connector_name, filename, expected_name): + assert SecretsLoader.generate_secret_name(connector_name, filename) == expected_name + + +def read_last_log_message(capfd): + _, err = capfd.readouterr() + print(err) + return err.split("# ")[-1].strip() + + +def test_main(capfd, monkeypatch): + # without parameters and envs + monkeypatch.delenv(ENV_GCP_GSM_CREDENTIALS, raising=False) + monkeypatch.setattr("sys.argv", [None, TEST_CONNECTOR_NAME, "fake_arg"]) + assert main() == 1 + assert "one script argument only" in read_last_log_message(capfd) + + monkeypatch.setattr("sys.argv", [None, TEST_CONNECTOR_NAME]) + # without env values + assert main() == 1 + assert "shouldn't be empty" in read_last_log_message(capfd) + + # incorrect GCP_GSM_CREDENTIALS + monkeypatch.setenv(ENV_GCP_GSM_CREDENTIALS, "non-json") + assert main() == 1 + assert "incorrect GCP_GSM_CREDENTIALS value" in read_last_log_message(capfd) + + # empty GCP_GSM_CREDENTIALS + monkeypatch.setenv(ENV_GCP_GSM_CREDENTIALS, "{}") + assert main() == 1 + assert "GCP_GSM_CREDENTIALS shouldn't be empty!" in read_last_log_message(capfd) + + # successful result + monkeypatch.setenv(ENV_GCP_GSM_CREDENTIALS, '{"test": "test"}') + + monkeypatch.setattr(SecretsLoader, "read_from_gsm", lambda *args, **kwargs: {}) + monkeypatch.setattr(SecretsLoader, "write_to_storage", lambda *args, **kwargs: 0) + assert main() == 0 + + +@pytest.mark.parametrize( + "connector_name,gsm_secrets,expected_secrets", + ( + ( + "source-gsm-only", + { + "config": {"test_key": "test_value"}, + "config_oauth": {"test_key_1": "test_key_2"}, + }, + [ + ("config.json", {"test_key": "test_value"}), + ("config_oauth.json", {"test_key_1": "test_key_2"}), + ] + ), + ), + ids=["gsm_only", ] + +) +@patch('ci_common_utils.GoogleApi.get_access_token', lambda *args: ("fake_token", None)) +@patch('ci_common_utils.GoogleApi.project_id', "fake_id") +def test_read(connector_name, gsm_secrets, expected_secrets): + matcher_gsm_list = re.compile("https://secretmanager.googleapis.com/v1/projects/.+/secrets") + secrets_list = {"secrets": [{ + "name": f"projects//secrets/SECRET_{connector_name.upper()}_{i}_CREDS", + "labels": { + "filename": k, + "connector": connector_name, + } + } for i, k in enumerate(gsm_secrets)]} + matcher_secret = re.compile("https://secretmanager.googleapis.com/v1/.+/versions/latest:access") + secrets_response_list = [{ + "json": {"payload": {"data": base64.b64encode(json.dumps(v).encode()).decode("utf-8")}} + } for v in gsm_secrets.values()] + + matcher_version = re.compile("https://secretmanager.googleapis.com/v1/.+:addVersion") + loader = SecretsLoader(connector_name=connector_name, gsm_credentials={}) + with requests_mock.Mocker() as m: + m.get(matcher_gsm_list, json=secrets_list) + m.post(matcher_gsm_list, json={"name": ""}) + m.post(matcher_version, json={}) + m.get(matcher_secret, secrets_response_list) + + secrets = [(*k, v.replace(" ", "")) for k, v in loader.read_from_gsm().items()] + expected_secrets = [(connector_name, k[0], json.dumps(k[1]).replace(" ", "")) for k in expected_secrets] + # raise Exception("%s => %s" % (secrets, expected_secrets)) + # raise Exception(set(secrets).symmetric_difference(set(expected_secrets))) + assert not set(secrets).symmetric_difference(set(expected_secrets)) + + +@pytest.mark.parametrize( + "connector_name,secrets,expected_files", + ( + ("source-test", {"test.json": "test_value"}, + ["airbyte-integrations/connectors/source-test/secrets/test.json"]), + + ("source-test2", {"test.json": "test_value", "auth.json": "test_auth"}, + ["airbyte-integrations/connectors/source-test2/secrets/test.json", + "airbyte-integrations/connectors/source-test2/secrets/auth.json"]), + + ("base-normalization", {"test.json": "test_value", "auth.json": "test_auth"}, + ["airbyte-integrations/bases/base-normalization/secrets/test.json", + "airbyte-integrations/bases/base-normalization/secrets/auth.json"]), + ), + ids=["single", "multi", "base-normalization"], +) +def test_write(connector_name, secrets, expected_files): + loader = SecretsLoader(connector_name=connector_name, gsm_credentials={}) + loader.base_folder = TEMP_FOLDER + loader.write_to_storage({(connector_name, k): v for k, v in secrets.items()}) + for expected_file in expected_files: + target_file = TEMP_FOLDER / expected_file + assert target_file.exists() + has = False + for k, v in secrets.items(): + if target_file.name == k: + with open(target_file, "r") as f: + assert f.read() == v + has = True + break + assert has, f"incorrect file data: {target_file}" diff --git a/tools/lib/gcp-token.sh b/tools/lib/gcp-token.sh deleted file mode 100644 index 00ac396ba50b..000000000000 --- a/tools/lib/gcp-token.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# Test script to access/generate secrets in Secret Manager - -# PROJECT="engineering-devops" -# SCOPE="https://www.googleapis.com/auth/cloud-platform" -# SERVICE_ACCOUNT_FILE=secret-manager.json -# SECRET=my-secret -TOKEN_TTL=3600 - - -_var2base64() { - printf "$1" | _urlencode_base64 -} - -_urlencode_base64() { - base64 | tr '/+' '_-' | tr -d '=\n' -} - -function _parse_token_uri(){ - local config_file=$1 - local token_uri=$(jq -r .token_uri ${config_file}) - echo "${token_uri}" -} - -function _generate_jwt() { - # Generate JWT token by a service account json file and scopes - local config_file=$1 - local scopes=$2 - - local now="$(date +%s)" - local expiration_time=$((${now} + ${TOKEN_TTL})) - # parse a file with credentials - local private_key=$(jq -r .private_key ${config_file}) - local client_email=$(jq -r .client_email ${config_file}) - local token_uri=$(_parse_token_uri "${config_file}") - - local claim=$(echo "{ - \"iat\": ${now}, - \"iss\": \"${client_email}\", - \"scope\": \"$scopes\", - \"aud\": \"${token_uri}\", - \"exp\":${expiration_time} - }" | jq -c) - local headers='{"typ":"JWT","alg":"RS256"}' - local body="$(_var2base64 "$headers").$(_var2base64 "$claim")" - local signature=$(openssl dgst -sha256 -sign <(echo "$private_key") <(printf "$body") | _urlencode_base64) - echo "$body.$signature" -} - -function parse_project_id(){ - # find a project_id into config file - local config_file=$1 - local project_id=$(jq -r .project_id ${config_file}) - echo "${project_id}" -} - -function get_gcp_access_token() { - # Generate an access token by a service account json file and scopes - local config_file="$1" - local scopes="$2" - local jwt=`_generate_jwt "${config_file}" "$scopes"` - local token_uri=$(_parse_token_uri "${config_file}") - local data=$(curl -s -X POST ${token_uri} \ - --data-urlencode "assertion=${jwt}" \ - --data-urlencode 'grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer' - ) - echo $data | jq -r .access_token -} - diff --git a/tools/python/.flake8 b/tools/python/.flake8 index d2c341f459c0..a270aeef0863 100644 --- a/tools/python/.flake8 +++ b/tools/python/.flake8 @@ -3,6 +3,8 @@ exclude = .venv, models # generated protocol models .eggs # python libraries" + .tox + build extend-ignore = E203, # whitespace before ':' (conflicts with Black) E231, # Bad trailing comma (conflicts with Black) diff --git a/tools/tox_ci.ini b/tools/tox_ci.ini new file mode 100644 index 000000000000..97c84c5921a7 --- /dev/null +++ b/tools/tox_ci.ini @@ -0,0 +1,37 @@ +[tox] +minversion = 1.9 +skipsdist = True +recreate = True + +envlist = + # list of all CI packages + ci_common_utils + ci_credentials + + +[base] +deps = + -e{toxinidir}/{envname}[tests] + pytest~=6.2.5 + flake8 + +[testenv] +# required for the `commands`. +changedir = {toxinidir}/{envname} +setupdir = {toxinidir} +# setupdir = {toxinidir}/{envname}:{toxinidir} +usedevelop = False + +deps = + {[base]deps} +setenv = + PYTHONPATH = {toxinidir}/{envname}:{toxinidir}/ci_common_utils + +# add the quiet option +install_command = pip --quiet install {opts} {packages} + +commands = + flake8 --config {toxinidir}/python/.flake8 {toxinidir}/{envname} + pytest + +