diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 35b495d33b47..71da55d06332 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -70,142 +70,10 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user - name: Write Integration Test Credentials # TODO DRY this with test-command.yml - run: ./tools/bin/ci_credentials.sh + run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }} env: - AMAZON_SELLER_PARTNER_TEST_CREDS: ${{ secrets.AMAZON_SELLER_PARTNER_TEST_CREDS }} - AMAZON_ADS_TEST_CREDS: ${{ secrets.AMAZON_ADS_TEST_CREDS }} - AMPLITUDE_INTEGRATION_TEST_CREDS: ${{ secrets.AMPLITUDE_INTEGRATION_TEST_CREDS }} - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} - AWS_REDSHIFT_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_REDSHIFT_INTEGRATION_TEST_CREDS }} - AWS_ORACLE_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_ORACLE_INTEGRATION_TEST_CREDS }} - SOURCE_AWS_CLOUDTRAIL_CREDS: ${{ secrets.SOURCE_AWS_CLOUDTRAIL_CREDS }} - AZURE_STORAGE_INTEGRATION_TEST_CREDS: ${{ secrets.AZURE_STORAGE_INTEGRATION_TEST_CREDS }} - BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }} - BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS }} - SOURCE_BING_ADS_CREDS: ${{ secrets.SOURCE_BING_ADS_CREDS }} - BIGQUERY_TEST_CREDS: ${{ secrets.BIGQUERY_TEST_CREDS }} - BRAINTREE_TEST_CREDS: ${{ secrets.BRAINTREE_TEST_CREDS }} - CART_TEST_CREDS: ${{ secrets.CART_TEST_CREDS }} - CHARGEBEE_INTEGRATION_TEST_CREDS: ${{ secrets.CHARGEBEE_INTEGRATION_TEST_CREDS }} - DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS }} - DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS }} - DESTINATION_PUBSUB_TEST_CREDS: ${{ secrets.DESTINATION_PUBSUB_TEST_CREDS }} - DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }} - DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }} - DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }} - SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }} - EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }} - FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }} - FACEBOOK_PAGES_INTEGRATION_TEST_CREDS: ${{ secrets.FACEBOOK_PAGES_INTEGRATION_TEST_CREDS }} - FILE_SECURE_HTTPS_TEST_CREDS: ${{ secrets.FILE_SECURE_HTTPS_TEST_CREDS }} - FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} - GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} - GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }} - GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} - GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }} - GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC }} - GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} - GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} - GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }} - GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} - GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }} - GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} - GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC }} - GOOGLE_SHEETS_TESTS_CREDS_OLD: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_OLD }} - GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }} - GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }} - GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }} - HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }} - HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} - HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH }} - INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }} - INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }} - INTERCOM_INTEGRATION_OAUTH_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_OAUTH_TEST_CREDS }} - ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }} - JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} - KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }} - LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }} - LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }} - MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} - MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} - MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }} - MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} - PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} - PINTEREST_TEST_CREDS: ${{ secrets.PINTEREST_TEST_CREDS }} - POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }} - POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }} - MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }} - MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }} - POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} - PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} - PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH }} - PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD }} - RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} - QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} - SALESFORCE_BULK_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_BULK_INTEGRATION_TESTS_CREDS }} - SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} - SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }} - SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }} - SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }} - SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }} - SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }} - SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }} - SOURCE_SLACK_OAUTH_TEST_CREDS: ${{ secrets.SOURCE_SLACK_OAUTH_TEST_CREDS }} - SOURCE_US_CENSUS_TEST_CREDS: ${{ secrets.SOURCE_US_CENSUS_TEST_CREDS }} - SMARTSHEETS_TEST_CREDS: ${{ secrets.SMARTSHEETS_TEST_CREDS }} - SOURCE_SNAPCHAT_MARKETING_CREDS: ${{ secrets.SOURCE_SNAPCHAT_MARKETING_CREDS }} - SNOWFLAKE_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDS }} - SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }} - SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }} - SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }} - SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }} - SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }} - SOURCE_S3_TEST_CREDS: ${{ secrets.SOURCE_S3_TEST_CREDS }} - SOURCE_S3_PARQUET_CREDS: ${{ secrets.SOURCE_S3_PARQUET_CREDS }} - SOURCE_SHORTIO_TEST_CREDS: ${{ secrets.SOURCE_SHORTIO_TEST_CREDS }} - SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }} - STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }} - SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }} - TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }} - TRELLO_TEST_CREDS: ${{ secrets.TRELLO_TEST_CREDS }} - TWILIO_TEST_CREDS: ${{ secrets.TWILIO_TEST_CREDS }} - SOURCE_TYPEFORM_CREDS: ${{ secrets.SOURCE_TYPEFORM_CREDS }} - ZENDESK_CHAT_INTEGRATION_TEST_CREDS: ${{ secrets.ZENDESK_CHAT_INTEGRATION_TEST_CREDS }} - ZENDESK_SUNSHINE_TEST_CREDS: ${{ secrets.ZENDESK_SUNSHINE_TEST_CREDS }} - ZENDESK_TALK_TEST_CREDS: ${{ secrets.ZENDESK_TALK_TEST_CREDS }} - ZENDESK_SUPPORT_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_TEST_CREDS }} - ZENDESK_SUPPORT_OAUTH_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_OAUTH_TEST_CREDS }} - ZOOM_INTEGRATION_TEST_CREDS: ${{ secrets.ZOOM_INTEGRATION_TEST_CREDS }} - PLAID_INTEGRATION_TEST_CREDS: ${{ secrets.PLAID_INTEGRATION_TEST_CREDS }} - DESTINATION_S3_INTEGRATION_TEST_CREDS: ${{ secrets.DESTINATION_S3_INTEGRATION_TEST_CREDS }} - DESTINATION_AZURE_BLOB_CREDS: ${{ secrets.DESTINATION_AZURE_BLOB_CREDS }} - DESTINATION_GCS_CREDS: ${{ secrets.DESTINATION_GCS_CREDS }} - APIFY_INTEGRATION_TEST_CREDS: ${{ secrets.APIFY_INTEGRATION_TEST_CREDS }} - DESTINATION_DYNAMODB_TEST_CREDS: ${{ secrets.DESTINATION_DYNAMODB_TEST_CREDS }} - SOURCE_ZUORA_TEST_CREDS: ${{ secrets.SOURCE_ZUORA_TEST_CREDS }} - SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }} - SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }} - SOURCE_LINKEDIN_ADS_TEST_CREDS: ${{ secrets.SOURCE_LINKEDIN_ADS_TEST_CREDS }} - SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }} - SOURCE_TIKTOK_MARKETING_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_TEST_CREDS }} - SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS }} - DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} - MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }} - SOURCE_ONESIGNAL_TEST_CREDS: ${{ secrets.SOURCE_ONESIGNAL_TEST_CREDS }} - SOURCE_SALESLOFT_TEST_CREDS: ${{ secrets.SOURCE_SALESLOFT_TEST_CREDS }} - SOURCE_CONFLUENCE_TEST_CREDS: ${{ secrets.SOURCE_CONFLUENCE_TEST_CREDS }} - SOURCE_AMAZON_SQS_TEST_CREDS: ${{ secrets.SOURCE_AMAZON_SQS_TEST_CREDS }} - SOURCE_FRESHSERVICE_TEST_CREDS: ${{ secrets.SOURCE_FRESHSERVICE_TEST_CREDS }} - SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }} - SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }} - SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }} - SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }} - SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }} - SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }} - SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }} - SOURCE_MONDAY_TEST_CREDS: ${{ secrets.SOURCE_MONDAY_TEST_CREDS }} - SOURCE_COMMERCETOOLS_TEST_CREDS: ${{ secrets.SOURCE_COMMERCETOOLS_TEST_CREDS }} + GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }} + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD} ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 976b2fdfc9e7..108bb682e761 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -65,142 +65,10 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user - name: Write Integration Test Credentials - run: ./tools/bin/ci_credentials.sh + run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }} env: - AMAZON_SELLER_PARTNER_TEST_CREDS: ${{ secrets.AMAZON_SELLER_PARTNER_TEST_CREDS }} - AMAZON_ADS_TEST_CREDS: ${{ secrets.AMAZON_ADS_TEST_CREDS }} - AMPLITUDE_INTEGRATION_TEST_CREDS: ${{ secrets.AMPLITUDE_INTEGRATION_TEST_CREDS }} - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} - AWS_ORACLE_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_ORACLE_INTEGRATION_TEST_CREDS }} - SOURCE_AWS_CLOUDTRAIL_CREDS: ${{ secrets.SOURCE_AWS_CLOUDTRAIL_CREDS }} - AWS_REDSHIFT_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_REDSHIFT_INTEGRATION_TEST_CREDS }} - AZURE_STORAGE_INTEGRATION_TEST_CREDS: ${{ secrets.AZURE_STORAGE_INTEGRATION_TEST_CREDS }} - BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }} - BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS }} - SOURCE_BING_ADS_CREDS: ${{ secrets.SOURCE_BING_ADS_CREDS }} - BIGQUERY_TEST_CREDS: ${{ secrets.BIGQUERY_TEST_CREDS }} - BRAINTREE_TEST_CREDS: ${{ secrets.BRAINTREE_TEST_CREDS }} - CART_TEST_CREDS: ${{ secrets.CART_TEST_CREDS }} - CHARGEBEE_INTEGRATION_TEST_CREDS: ${{ secrets.CHARGEBEE_INTEGRATION_TEST_CREDS }} - DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS }} - DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS }} - DESTINATION_PUBSUB_TEST_CREDS: ${{ secrets.DESTINATION_PUBSUB_TEST_CREDS }} - DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }} - DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }} - DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }} - SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }} - EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }} - FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }} - FACEBOOK_PAGES_INTEGRATION_TEST_CREDS: ${{ secrets.FACEBOOK_PAGES_INTEGRATION_TEST_CREDS }} - FILE_SECURE_HTTPS_TEST_CREDS: ${{ secrets.FILE_SECURE_HTTPS_TEST_CREDS }} - FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} - GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} - GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }} - GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} - GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }} - GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC }} - GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }} - GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }} - GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }} - GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }} - GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }} - GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} - GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC }} - GOOGLE_SHEETS_TESTS_CREDS_OLD: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_OLD }} - GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }} - GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }} - GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }} - HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }} - HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} - HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH }} - INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }} - INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }} - INTERCOM_INTEGRATION_OAUTH_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_OAUTH_TEST_CREDS }} - ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }} - JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} - KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }} - SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }} - LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }} - LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }} - MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} - MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} - MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }} - MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} - PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} - PINTEREST_TEST_CREDS: ${{ secrets.PINTEREST_TEST_CREDS }} - POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }} - POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }} - MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }} - MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }} - POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} - PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} - PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH }} - PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD }} - RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} - QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} - SALESFORCE_BULK_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_BULK_INTEGRATION_TESTS_CREDS }} - SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} - SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }} - SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }} - SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }} - SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }} - SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }} - SOURCE_SLACK_OAUTH_TEST_CREDS: ${{ secrets.SOURCE_SLACK_OAUTH_TEST_CREDS }} - SOURCE_US_CENSUS_TEST_CREDS: ${{ secrets.SOURCE_US_CENSUS_TEST_CREDS }} - SMARTSHEETS_TEST_CREDS: ${{ secrets.SMARTSHEETS_TEST_CREDS }} - SOURCE_SNAPCHAT_MARKETING_CREDS: ${{ secrets.SOURCE_SNAPCHAT_MARKETING_CREDS }} - SNOWFLAKE_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDS }} - SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }} - SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }} - SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }} - SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }} - SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }} - SOURCE_S3_TEST_CREDS: ${{ secrets.SOURCE_S3_TEST_CREDS }} - SOURCE_S3_PARQUET_CREDS: ${{ secrets.SOURCE_S3_PARQUET_CREDS }} - SOURCE_SHORTIO_TEST_CREDS: ${{ secrets.SOURCE_SHORTIO_TEST_CREDS }} - SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }} - STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }} - SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }} - TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }} - TRELLO_TEST_CREDS: ${{ secrets.TRELLO_TEST_CREDS }} - TWILIO_TEST_CREDS: ${{ secrets.TWILIO_TEST_CREDS }} - SOURCE_TYPEFORM_CREDS: ${{ secrets.SOURCE_TYPEFORM_CREDS }} - ZENDESK_CHAT_INTEGRATION_TEST_CREDS: ${{ secrets.ZENDESK_CHAT_INTEGRATION_TEST_CREDS }} - ZENDESK_SUNSHINE_TEST_CREDS: ${{ secrets.ZENDESK_SUNSHINE_TEST_CREDS }} - ZENDESK_TALK_TEST_CREDS: ${{ secrets.ZENDESK_TALK_TEST_CREDS }} - ZENDESK_SUPPORT_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_TEST_CREDS }} - ZENDESK_SUPPORT_OAUTH_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_OAUTH_TEST_CREDS }} - ZOOM_INTEGRATION_TEST_CREDS: ${{ secrets.ZOOM_INTEGRATION_TEST_CREDS }} - PLAID_INTEGRATION_TEST_CREDS: ${{ secrets.PLAID_INTEGRATION_TEST_CREDS }} - DESTINATION_S3_INTEGRATION_TEST_CREDS: ${{ secrets.DESTINATION_S3_INTEGRATION_TEST_CREDS }} - DESTINATION_AZURE_BLOB_CREDS: ${{ secrets.DESTINATION_AZURE_BLOB_CREDS }} - DESTINATION_GCS_CREDS: ${{ secrets.DESTINATION_GCS_CREDS }} - DESTINATION_DYNAMODB_TEST_CREDS: ${{ secrets.DESTINATION_DYNAMODB_TEST_CREDS }} - APIFY_INTEGRATION_TEST_CREDS: ${{ secrets.APIFY_INTEGRATION_TEST_CREDS }} - SOURCE_ZUORA_TEST_CREDS: ${{ secrets.SOURCE_ZUORA_TEST_CREDS }} - SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }} - SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }} - SOURCE_LINKEDIN_ADS_TEST_CREDS: ${{ secrets.SOURCE_LINKEDIN_ADS_TEST_CREDS }} - SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }} - SOURCE_TIKTOK_MARKETING_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_TEST_CREDS }} - SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS }} - DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} - MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }} - SOURCE_ONESIGNAL_TEST_CREDS: ${{ secrets.SOURCE_ONESIGNAL_TEST_CREDS }} - SOURCE_SALESLOFT_TEST_CREDS: ${{ secrets.SOURCE_SALESLOFT_TEST_CREDS }} - SOURCE_CONFLUENCE_TEST_CREDS: ${{ secrets.SOURCE_CONFLUENCE_TEST_CREDS }} - SOURCE_AMAZON_SQS_TEST_CREDS: ${{ secrets.SOURCE_AMAZON_SQS_TEST_CREDS }} - SOURCE_FRESHSERVICE_TEST_CREDS: ${{ secrets.SOURCE_FRESHSERVICE_TEST_CREDS }} - SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }} - SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }} - SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }} - SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }} - SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }} - SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }} - SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }} - SOURCE_MONDAY_TEST_CREDS: ${{ secrets.SOURCE_MONDAY_TEST_CREDS }} - SOURCE_COMMERCETOOLS_TEST_CREDS: ${{ secrets.SOURCE_COMMERCETOOLS_TEST_CREDS }} + GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }} + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: | ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} name: test ${{ github.event.inputs.connector }} diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index ebf9705e5784..0b9970ec6d47 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -130,10 +130,23 @@ Once you've finished iterating on the changes to a connector as specified in its ## Using credentials in CI In order to run integration tests in CI, you'll often need to inject credentials into CI. There are a few steps for doing this: - -1. **Place the credentials into Lastpass**: Airbyte uses a shared Lastpass account as the source of truth for all secrets. Place the credentials **exactly as they should be used by the connector** into a secure note i.e: it should basically be a copy paste of the `config.json` passed into a connector via the `--config` flag. We use the following naming pattern: ` creds` e.g: `source google adwords creds` or `destination snowflake creds`. -2. **Add the credentials to Github Secrets**: To inject credentials into a CI workflow, the first step is to add it to Github Secrets, specifically within the ["more-secrets" environment](https://github.com/airbytehq/airbyte/settings/environments/276695501/edit). Admin access to the Airbyte repo is required to do this. All Airbyte engineers have admin access and should be able to do this themselves. External contributors or contractors will need to request this from their team lead or project manager who should have admin access. Follow the same naming pattern as all the other secrets e.g: if you are placing credentials for source google adwords, name the secret `SOURCE_GOOGLE_ADWORDS_CREDS`. After doing this step, the secret will be available in the relevant Github workflows using the workflow secrets syntax. -3. **Inject the credentials into test and publish CI workflows**: edit the files `.github/workflows/publish-command.yml` and `.github/workflows/test-command.yml` to inject the secret into the CI run. This will make these secrets available to the `/test` and `/publish` commands. -4. **During CI, write the secret from env variables to the connector directory**: edit `tools/bin/ci_credentials.sh` to write the secret into the `secrets/` directory of the relevant connector. -5. That should be it. +1. **Place the credentials into Google Secret Manager(GSM)**: Airbyte uses a project 'Google Secret Manager' service as the source of truth for all CI secrets. Place the credentials **exactly as they should be used by the connector** into a GSM secret [here](https://console.cloud.google.com/security/secret-manager?referrer=search&orgonly=true&project=dataline-integration-testing&supportedpurview=organizationId) i.e.: it should basically be a copy paste of the `config.json` passed into a connector via the `--config` flag. We use the following naming pattern: `SECRET__CREDS` e.g: `SECRET_SOURCE-S3_CREDS` or `SECRET_DESTINATION-SNOWFLAKE_CREDS`. +2. **Add the GSM secret's labels**: + * `connector` (required) -- unique connector's name or set of connectors' names with '_' as delimiter i.e.: `connector=source-s3`, `connector=destination-snowflake` + * `filename` (optional) -- custom target secret file. Unfortunately Google doesn't use '.' into labels' values and so Airbyte CI scripts will add '.json' to the end automatically. By default secrets will be saved to `./secrets/config.json` i.e: `filename=config_auth` => `secrets/config_auth.json` +3. That should be it. + +#### Access CI secrets on GSM +Access to GSM storage is limited to Airbyte employees. To give an employee permissions to the project: +1. Go to the permissions' [page](https://console.cloud.google.com/iam-admin/iam?project=dataline-integration-testing) +2. Add a new principal to `dataline-integration-testing`: +- input their login email +- select the role `Development_CI_Secrets` +3. Save + +#### How to migrate to the new secrets' logic: +1. Create all necessary secrets according to the instructions above +2. Remove all lines with old connector's Github secrets from this file: tools/bin/ci_credentials.sh +3. Remove all old secrets from Github repository secrets. +4. That should be it. diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 95741a3bf8fe..01822588c87c 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -1,16 +1,54 @@ #!/usr/bin/env bash . tools/lib/lib.sh +. tools/lib/gcp-token.sh set -e -function write_standard_creds() { +# all secrets will be loaded if the second argument is not present +CONNECTOR_FULLNAME=${1:-all} +CONNECTOR_NAME=`echo ${CONNECTOR_FULLNAME} | rev | cut -d'/' -f1 | rev` + +GSM_SCOPES="https://www.googleapis.com/auth/cloud-platform" + +# If a secret is available in both Github and GSM, then the GSM secret is used, otherwise Github. + +declare -A SECRET_MAP + + +function read_secrets() { local connector_name=$1 local creds=$2 local cred_filename=${3:-config.json} + local secrets_provider_name=${4:-github} [ -z "$connector_name" ] && error "Empty connector name" - [ -z "$creds" ] && error "Creds not set for $connector_name" + [ -z "$creds" ] && echo "!!!!!Creds not set for the connector $connector_name from ${secrets_provider_name}" + + if [[ $CONNECTOR_NAME != "all" && ${connector_name} != ${CONNECTOR_NAME} ]]; then + return 0 + fi + local key="${connector_name}#${cred_filename}" + [[ -z "${creds}" ]] && error "Empty credential for the connector '${key} from ${secrets_provider_name}" + + if [ -v SECRET_MAP[${key}] ]; then + echo "The connector '${key}' was added before" + return 0 + fi + echo "register the secret ${key} from ${secrets_provider_name}" + SECRET_MAP[${key}]="${creds}" + return 0 +} + +function write_secret_to_disk() { + local connector_name=$1 + local cred_filename=$2 + local creds=$3 + if jq -e . >/dev/null 2>&1 <<< "${creds}"; then + echo "Parsed JSON for '${connector_name}' => ${cred_filename} successfully" + else + error "Failed to parse JSON for '${connector_name}' => ${cred_filename}" + fi if [ "$connector_name" = "base-normalization" ]; then local secrets_dir="airbyte-integrations/bases/${connector_name}/secrets" @@ -18,153 +56,246 @@ function write_standard_creds() { local secrets_dir="airbyte-integrations/connectors/${connector_name}/secrets" fi mkdir -p "$secrets_dir" + echo "Saved a secret => ${secrets_dir}/${cred_filename}" echo "$creds" > "${secrets_dir}/${cred_filename}" } +function write_all_secrets() { + for key in "${!SECRET_MAP[@]}"; do + local connector_name=$(echo ${key} | cut -d'#' -f1) + local cred_filename=$(echo ${key} | cut -d'#' -f2) + local creds=${SECRET_MAP[${key}]} + write_secret_to_disk ${connector_name} ${cred_filename} "${creds}" + + done + return 0 +} + + +function export_github_secrets(){ + # We expect that all secrets injected from github are available in an env variable `SECRETS_JSON` + local pairs=`echo ${GITHUB_PROVIDED_SECRETS_JSON} | jq -c 'keys[] as $k | {"name": $k, "value": .[$k]} | @base64'` + while read row; do + pair=$(echo "${row}" | tr -d '"' | base64 -d) + local key=$(echo ${pair} | jq -r .name) + local value=$(echo ${pair} | jq -r .value) + if [[ "$key" == *"_CREDS"* ]]; then + declare -gxr "${key}"="$(echo ${value})" + fi + done <<< ${pairs} + unset GITHUB_PROVIDED_SECRETS_JSON +} + +function export_gsm_secrets(){ + local config_file=`mktemp` + echo "${GCP_GSM_CREDENTIALS}" > ${config_file} + local access_token=$(get_gcp_access_token "${config_file}" "${GSM_SCOPES}") + local project_id=$(parse_project_id "${config_file}") + rm ${config_file} + + # docs: https://cloud.google.com/secret-manager/docs/filtering#api + local filter="name:SECRET_" + [[ ${CONNECTOR_NAME} != "all" ]] && filter="${filter} AND labels.connector:${CONNECTOR_NAME}" + local uri="https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets" + local next_token='' + while true; do + local data=$(curl -s --get --fail "${uri}" \ + --data-urlencode "filter=${filter}" \ + --data-urlencode "pageToken=${next_token}" \ + --header "authorization: Bearer ${access_token}" \ + --header "content-type: application/json" \ + --header "x-goog-user-project: ${project_id}") + [[ -z ${data} ]] && error "Can't load secret for connector ${CONNECTOR_NAME}" + # GSM returns an empty JSON object if secrets are not found. + # It breaks JSON parsing by the 'jq' utility. The simplest fix is response normalization + [[ ${data} == "{}" ]] && data='{"secrets": []}' + + for row in $(echo "${data}" | jq -r '.secrets[] | @base64'); do + local secret_info=$(echo ${row} | base64 --decode) + local secret_name=$(echo ${secret_info}| jq -r .name) + local label_filename=$(echo ${secret_info}| jq -r '.labels.filename // "config"') + local label_connectors=$(echo ${secret_info}| jq -r '.labels.connector // ""') + + # skip secrets without the label "connector" + [[ -z ${label_connectors} ]] && continue + if [[ "$label_connectors" != *"${CONNECTOR_NAME}"* ]]; then + echo "Not found ${CONNECTOR_NAME} info into the label 'connector' of the secret ${secret_name}" + continue + fi + + # all secret file names should be finished with ".json" + # but '.' cant be used in google, so we append it + local filename="${label_filename}.json" + echo "found the Google secret of ${label_connectors}: ${secret_name} => ${filename}" + local secret_uri="https://secretmanager.googleapis.com/v1/${secret_name}/versions/latest:access" + local secret_data=$(curl -s --get --fail "${secret_uri}" \ + --header "authorization: Bearer ${access_token}" \ + --header "content-type: application/json" \ + --header "x-goog-user-project: ${project_id}") + [[ -z ${secret_data} ]] && error "Can't load secrets' list" + + secret_data=$(echo ${secret_data} | jq -r '.payload.data // ""' | base64 -d) + read_secrets "${CONNECTOR_NAME}" "${secret_data}" "${filename}" "gsm" + done + next_token=`echo ${data} | jq -r '.nextPageToken // ""'` + [[ -z ${next_token} ]] && break + done + return 0 +} + +export_gsm_secrets +export_github_secrets + + + # Please maintain this organisation and alphabetise. -write_standard_creds destination-bigquery "$BIGQUERY_INTEGRATION_TEST_CREDS" "credentials.json" -write_standard_creds destination-bigquery-denormalized "$BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS" "credentials.json" -write_standard_creds destination-databricks "$DESTINATION_DATABRICKS_CREDS" -write_standard_creds destination-gcs "$DESTINATION_GCS_CREDS" -write_standard_creds destination-kvdb "$DESTINATION_KVDB_TEST_CREDS" -write_standard_creds destination-keen "$DESTINATION_KEEN_TEST_CREDS" - -write_standard_creds destination-postgres "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json" -write_standard_creds destination-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" -write_standard_creds destination-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json" -write_standard_creds destination-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" -write_standard_creds destination-pubsub "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json" -write_standard_creds destination-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" -write_standard_creds destination-dynamodb "$DESTINATION_DYNAMODB_TEST_CREDS" -write_standard_creds destination-oracle "$AWS_ORACLE_INTEGRATION_TEST_CREDS" -write_standard_creds destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS" -write_standard_creds destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS" -write_standard_creds destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json" -write_standard_creds destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json" -write_standard_creds destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json" - -write_standard_creds base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json" -write_standard_creds base-normalization "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "snowflake.json" -write_standard_creds base-normalization "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" "redshift.json" -write_standard_creds base-normalization "$AWS_ORACLE_INTEGRATION_TEST_CREDS" "oracle.json" - -write_standard_creds source-amazon-seller-partner "$AMAZON_SELLER_PARTNER_TEST_CREDS" -write_standard_creds source-amazon-sqs "$SOURCE_AMAZON_SQS_TEST_CREDS" -write_standard_creds source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS" -write_standard_creds source-apify-dataset "$APIFY_INTEGRATION_TEST_CREDS" -write_standard_creds source-amazon-ads "$AMAZON_ADS_TEST_CREDS" -write_standard_creds source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS" -write_standard_creds source-asana "$SOURCE_ASANA_TEST_CREDS" -write_standard_creds source-aws-cloudtrail "$SOURCE_AWS_CLOUDTRAIL_CREDS" -write_standard_creds source-bamboo-hr "$SOURCE_BAMBOO_HR_CREDS" -write_standard_creds source-bigcommerce "$SOURCE_BIGCOMMERCE_CREDS" -write_standard_creds source-bigquery "$BIGQUERY_TEST_CREDS" "credentials.json" -write_standard_creds source-bing-ads "$SOURCE_BING_ADS_CREDS" -write_standard_creds source-braintree "$BRAINTREE_TEST_CREDS" -write_standard_creds source-cart "$CART_TEST_CREDS" -write_standard_creds source-chargebee "$CHARGEBEE_INTEGRATION_TEST_CREDS" -write_standard_creds source-close-com "$SOURCE_CLOSE_COM_CREDS" -write_standard_creds source-commercetools "$SOURCE_COMMERCETOOLS_TEST_CREDS" -write_standard_creds source-confluence "$SOURCE_CONFLUENCE_TEST_CREDS" -write_standard_creds source-delighted "$SOURCE_DELIGHTED_TEST_CREDS" -write_standard_creds source-drift "$DRIFT_INTEGRATION_TEST_CREDS" -write_standard_creds source-dixa "$SOURCE_DIXA_TEST_CREDS" -write_standard_creds source-exchange-rates "$EXCHANGE_RATES_TEST_CREDS" -write_standard_creds source-file "$GOOGLE_CLOUD_STORAGE_TEST_CREDS" "gcs.json" -write_standard_creds source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json" -write_standard_creds source-file "$AZURE_STORAGE_INTEGRATION_TEST_CREDS" "azblob.json" -write_standard_creds source-file "$FILE_SECURE_HTTPS_TEST_CREDS" -write_standard_creds source-file-secure "$FILE_SECURE_HTTPS_TEST_CREDS" -write_standard_creds source-freshdesk "$FRESHDESK_TEST_CREDS" -write_standard_creds source-freshsales "$SOURCE_FRESHSALES_TEST_CREDS" -write_standard_creds source-freshservice "$SOURCE_FRESHSERVICE_TEST_CREDS" -write_standard_creds source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" -write_standard_creds source-facebook-pages "$FACEBOOK_PAGES_INTEGRATION_TEST_CREDS" -write_standard_creds source-gitlab "$GITLAB_INTEGRATION_TEST_CREDS" -write_standard_creds source-github "$GH_NATIVE_INTEGRATION_TEST_CREDS" -write_standard_creds source-google-ads "$GOOGLE_ADS_TEST_CREDS" -write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS" -write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json" -write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json" -write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS" -write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json" -write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS" -write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json" -write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS" -write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC" "service_config.json" -write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.json" -write_standard_creds source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS" -write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS" -write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" -write_standard_creds source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS" -write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" -write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" -write_standard_creds source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" -write_standard_creds source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS" -write_standard_creds source-intercom "$INTERCOM_INTEGRATION_OAUTH_TEST_CREDS" "config_apikey.json" -write_standard_creds source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS" -write_standard_creds source-jira "$JIRA_INTEGRATION_TEST_CREDS" -write_standard_creds source-klaviyo "$KLAVIYO_TEST_CREDS" -write_standard_creds source-lemlist "$SOURCE_LEMLIST_TEST_CREDS" -write_standard_creds source-lever-hiring "$LEVER_HIRING_INTEGRATION_TEST_CREDS" -write_standard_creds source-looker "$LOOKER_INTEGRATION_TEST_CREDS" -write_standard_creds source-linkedin-ads "$SOURCE_LINKEDIN_ADS_TEST_CREDS" -write_standard_creds source-mailchimp "$MAILCHIMP_TEST_CREDS" -write_standard_creds source-marketo "$SOURCE_MARKETO_TEST_CREDS" -write_standard_creds source-microsoft-teams "$MICROSOFT_TEAMS_TEST_CREDS" -write_standard_creds source-mixpanel "$MIXPANEL_INTEGRATION_TEST_CREDS" -write_standard_creds source-monday "$SOURCE_MONDAY_TEST_CREDS" -write_standard_creds source-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" -write_standard_creds source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json" -write_standard_creds source-mssql "$MSSQL_RDS_TEST_CREDS" -write_standard_creds source-okta "$SOURCE_OKTA_TEST_CREDS" -write_standard_creds source-onesignal "$SOURCE_ONESIGNAL_TEST_CREDS" -write_standard_creds source-plaid "$PLAID_INTEGRATION_TEST_CREDS" -write_standard_creds source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS" -write_standard_creds source-pinterest "$PINTEREST_TEST_CREDS" -write_standard_creds source-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json" -write_standard_creds source-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" -write_standard_creds source-posthog "$POSTHOG_TEST_CREDS" -write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS" "config.json" -write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH" "oauth_config.json" -write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD" "old_config.json" -write_standard_creds source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS" -write_standard_creds source-recharge "$RECHARGE_INTEGRATION_TEST_CREDS" -write_standard_creds source-recurly "$SOURCE_RECURLY_INTEGRATION_TEST_CREDS" -write_standard_creds source-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" -write_standard_creds source-retently "$SOURCE_RETENTLY_TEST_CREDS" -write_standard_creds source-s3 "$SOURCE_S3_TEST_CREDS" -write_standard_creds source-s3 "$SOURCE_S3_PARQUET_CREDS" "parquet_config.json" -write_standard_creds source-salesforce "$SALESFORCE_BULK_INTEGRATION_TESTS_CREDS" "config_bulk.json" -write_standard_creds source-salesforce "$SALESFORCE_INTEGRATION_TESTS_CREDS" -write_standard_creds source-salesloft "$SOURCE_SALESLOFT_TEST_CREDS" -write_standard_creds source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS" -write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_CREDS" -write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json" -write_standard_creds source-shortio "$SOURCE_SHORTIO_TEST_CREDS" -write_standard_creds source-slack "$SOURCE_SLACK_TEST_CREDS" -write_standard_creds source-slack "$SOURCE_SLACK_OAUTH_TEST_CREDS" "config_oauth.json" -write_standard_creds source-smartsheets "$SMARTSHEETS_TEST_CREDS" -write_standard_creds source-snapchat-marketing "$SOURCE_SNAPCHAT_MARKETING_CREDS" -write_standard_creds source-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "config.json" -write_standard_creds source-square "$SOURCE_SQUARE_CREDS" -write_standard_creds source-strava "$SOURCE_STRAVA_TEST_CREDS" -write_standard_creds source-paystack "$SOURCE_PAYSTACK_TEST_CREDS" -write_standard_creds source-sentry "$SOURCE_SENTRY_TEST_CREDS" -write_standard_creds source-stripe "$SOURCE_STRIPE_CREDS" -write_standard_creds source-stripe "$STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS" "connected_account_config.json" -write_standard_creds source-surveymonkey "$SURVEYMONKEY_TEST_CREDS" -write_standard_creds source-tempo "$TEMPO_INTEGRATION_TEST_CREDS" -write_standard_creds source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_TEST_CREDS" -write_standard_creds source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS" "prod_config.json" -write_standard_creds source-trello "$TRELLO_TEST_CREDS" -write_standard_creds source-twilio "$TWILIO_TEST_CREDS" -write_standard_creds source-typeform "$SOURCE_TYPEFORM_CREDS" -write_standard_creds source-us-census "$SOURCE_US_CENSUS_TEST_CREDS" -write_standard_creds source-zendesk-chat "$ZENDESK_CHAT_INTEGRATION_TEST_CREDS" -write_standard_creds source-zendesk-sunshine "$ZENDESK_SUNSHINE_TEST_CREDS" -write_standard_creds source-zendesk-support "$ZENDESK_SUPPORT_TEST_CREDS" -write_standard_creds source-zendesk-support "$ZENDESK_SUPPORT_OAUTH_TEST_CREDS" "config_oauth.json" -write_standard_creds source-zendesk-talk "$ZENDESK_TALK_TEST_CREDS" -write_standard_creds source-zoom-singer "$ZOOM_INTEGRATION_TEST_CREDS" -write_standard_creds source-zuora "$SOURCE_ZUORA_TEST_CREDS" +read_secrets destination-bigquery "$BIGQUERY_INTEGRATION_TEST_CREDS" "credentials.json" +read_secrets destination-bigquery-denormalized "$BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS" "credentials.json" +read_secrets destination-databricks "$DESTINATION_DATABRICKS_CREDS" +read_secrets destination-gcs "$DESTINATION_GCS_CREDS" +read_secrets destination-kvdb "$DESTINATION_KVDB_TEST_CREDS" +read_secrets destination-keen "$DESTINATION_KEEN_TEST_CREDS" + +read_secrets destination-postgres "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json" +read_secrets destination-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" +read_secrets destination-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json" +read_secrets destination-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" +read_secrets destination-pubsub "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json" +read_secrets destination-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" +read_secrets destination-dynamodb "$DESTINATION_DYNAMODB_TEST_CREDS" +read_secrets destination-oracle "$AWS_ORACLE_INTEGRATION_TEST_CREDS" +read_secrets destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS" +read_secrets destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS" +read_secrets destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json" +read_secrets destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json" +read_secrets destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json" + +read_secrets base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json" +read_secrets base-normalization "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "snowflake.json" +read_secrets base-normalization "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" "redshift.json" +read_secrets base-normalization "$AWS_ORACLE_INTEGRATION_TEST_CREDS" "oracle.json" + +read_secrets source-amazon-seller-partner "$AMAZON_SELLER_PARTNER_TEST_CREDS" +read_secrets source-amazon-sqs "$SOURCE_AMAZON_SQS_TEST_CREDS" +read_secrets source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS" +read_secrets source-apify-dataset "$APIFY_INTEGRATION_TEST_CREDS" +read_secrets source-amazon-ads "$AMAZON_ADS_TEST_CREDS" +read_secrets source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS" +read_secrets source-asana "$SOURCE_ASANA_TEST_CREDS" +read_secrets source-aws-cloudtrail "$SOURCE_AWS_CLOUDTRAIL_CREDS" +read_secrets source-bamboo-hr "$SOURCE_BAMBOO_HR_CREDS" +read_secrets source-bigcommerce "$SOURCE_BIGCOMMERCE_CREDS" +read_secrets source-bigquery "$BIGQUERY_TEST_CREDS" "credentials.json" +read_secrets source-bing-ads "$SOURCE_BING_ADS_CREDS" +read_secrets source-braintree "$BRAINTREE_TEST_CREDS" +read_secrets source-cart "$CART_TEST_CREDS" +read_secrets source-chargebee "$CHARGEBEE_INTEGRATION_TEST_CREDS" +read_secrets source-close-com "$SOURCE_CLOSE_COM_CREDS" +read_secrets source-commercetools "$SOURCE_COMMERCETOOLS_TEST_CREDS" +read_secrets source-confluence "$SOURCE_CONFLUENCE_TEST_CREDS" +read_secrets source-delighted "$SOURCE_DELIGHTED_TEST_CREDS" +read_secrets source-drift "$DRIFT_INTEGRATION_TEST_CREDS" +read_secrets source-dixa "$SOURCE_DIXA_TEST_CREDS" +read_secrets source-exchange-rates "$EXCHANGE_RATES_TEST_CREDS" +read_secrets source-file "$GOOGLE_CLOUD_STORAGE_TEST_CREDS" "gcs.json" +read_secrets source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json" +read_secrets source-file "$AZURE_STORAGE_INTEGRATION_TEST_CREDS" "azblob.json" +read_secrets source-file "$FILE_SECURE_HTTPS_TEST_CREDS" +read_secrets source-file-secure "$FILE_SECURE_HTTPS_TEST_CREDS" +read_secrets source-freshdesk "$FRESHDESK_TEST_CREDS" +read_secrets source-freshsales "$SOURCE_FRESHSALES_TEST_CREDS" +read_secrets source-freshservice "$SOURCE_FRESHSERVICE_TEST_CREDS" +read_secrets source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" +read_secrets source-facebook-pages "$FACEBOOK_PAGES_INTEGRATION_TEST_CREDS" +read_secrets source-gitlab "$GITLAB_INTEGRATION_TEST_CREDS" +read_secrets source-github "$GH_NATIVE_INTEGRATION_TEST_CREDS" +read_secrets source-google-ads "$GOOGLE_ADS_TEST_CREDS" +read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS" +read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json" +read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json" +read_secrets source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS" +read_secrets source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json" +read_secrets source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS" +read_secrets source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json" +read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS" +read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC" "service_config.json" +read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.json" +read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS" +read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS" +read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" +read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS" +read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" +read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" +read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" +read_secrets source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS" +read_secrets source-intercom "$INTERCOM_INTEGRATION_OAUTH_TEST_CREDS" "config_apikey.json" +read_secrets source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS" +read_secrets source-jira "$JIRA_INTEGRATION_TEST_CREDS" +read_secrets source-klaviyo "$KLAVIYO_TEST_CREDS" +read_secrets source-lemlist "$SOURCE_LEMLIST_TEST_CREDS" +read_secrets source-lever-hiring "$LEVER_HIRING_INTEGRATION_TEST_CREDS" +read_secrets source-looker "$LOOKER_INTEGRATION_TEST_CREDS" +read_secrets source-linkedin-ads "$SOURCE_LINKEDIN_ADS_TEST_CREDS" +read_secrets source-mailchimp "$MAILCHIMP_TEST_CREDS" +read_secrets source-marketo "$SOURCE_MARKETO_TEST_CREDS" +read_secrets source-microsoft-teams "$MICROSOFT_TEAMS_TEST_CREDS" +read_secrets source-mixpanel "$MIXPANEL_INTEGRATION_TEST_CREDS" +read_secrets source-monday "$SOURCE_MONDAY_TEST_CREDS" +read_secrets source-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" +read_secrets source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json" +read_secrets source-mssql "$MSSQL_RDS_TEST_CREDS" +read_secrets source-okta "$SOURCE_OKTA_TEST_CREDS" +read_secrets source-onesignal "$SOURCE_ONESIGNAL_TEST_CREDS" +read_secrets source-plaid "$PLAID_INTEGRATION_TEST_CREDS" +read_secrets source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS" +read_secrets source-pinterest "$PINTEREST_TEST_CREDS" +read_secrets source-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json" +read_secrets source-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json" +read_secrets source-posthog "$POSTHOG_TEST_CREDS" +read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS" "config.json" +read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH" "oauth_config.json" +read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD" "old_config.json" +read_secrets source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS" +read_secrets source-recharge "$RECHARGE_INTEGRATION_TEST_CREDS" +read_secrets source-recurly "$SOURCE_RECURLY_INTEGRATION_TEST_CREDS" +read_secrets source-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" +read_secrets source-retently "$SOURCE_RETENTLY_TEST_CREDS" +read_secrets source-s3 "$SOURCE_S3_TEST_CREDS" +read_secrets source-s3 "$SOURCE_S3_PARQUET_CREDS" "parquet_config.json" +read_secrets source-salesforce "$SALESFORCE_BULK_INTEGRATION_TESTS_CREDS" "config_bulk.json" +read_secrets source-salesforce "$SALESFORCE_INTEGRATION_TESTS_CREDS" +read_secrets source-salesloft "$SOURCE_SALESLOFT_TEST_CREDS" +read_secrets source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS" +read_secrets source-shopify "$SHOPIFY_INTEGRATION_TEST_CREDS" +read_secrets source-shopify "$SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json" +read_secrets source-shortio "$SOURCE_SHORTIO_TEST_CREDS" +read_secrets source-slack "$SOURCE_SLACK_TEST_CREDS" +read_secrets source-slack "$SOURCE_SLACK_OAUTH_TEST_CREDS" "config_oauth.json" +read_secrets source-smartsheets "$SMARTSHEETS_TEST_CREDS" +read_secrets source-snapchat-marketing "$SOURCE_SNAPCHAT_MARKETING_CREDS" +read_secrets source-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "config.json" +read_secrets source-square "$SOURCE_SQUARE_CREDS" +read_secrets source-strava "$SOURCE_STRAVA_TEST_CREDS" +read_secrets source-paystack "$SOURCE_PAYSTACK_TEST_CREDS" +read_secrets source-sentry "$SOURCE_SENTRY_TEST_CREDS" +read_secrets source-stripe "$SOURCE_STRIPE_CREDS" +read_secrets source-stripe "$STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS" "connected_account_config.json" +read_secrets source-surveymonkey "$SURVEYMONKEY_TEST_CREDS" +read_secrets source-tempo "$TEMPO_INTEGRATION_TEST_CREDS" +read_secrets source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_TEST_CREDS" +read_secrets source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS" "prod_config.json" +read_secrets source-trello "$TRELLO_TEST_CREDS" +read_secrets source-twilio "$TWILIO_TEST_CREDS" +read_secrets source-typeform "$SOURCE_TYPEFORM_CREDS" +read_secrets source-us-census "$SOURCE_US_CENSUS_TEST_CREDS" +read_secrets source-zendesk-chat "$ZENDESK_CHAT_INTEGRATION_TEST_CREDS" +read_secrets source-zendesk-sunshine "$ZENDESK_SUNSHINE_TEST_CREDS" +read_secrets source-zendesk-support "$ZENDESK_SUPPORT_TEST_CREDS" +read_secrets source-zendesk-support "$ZENDESK_SUPPORT_OAUTH_TEST_CREDS" "config_oauth.json" +read_secrets source-zendesk-talk "$ZENDESK_TALK_TEST_CREDS" +read_secrets source-zoom-singer "$ZOOM_INTEGRATION_TEST_CREDS" +read_secrets source-zuora "$SOURCE_ZUORA_TEST_CREDS" + +write_all_secrets +exit $? + diff --git a/tools/lib/gcp-token.sh b/tools/lib/gcp-token.sh new file mode 100644 index 000000000000..00ac396ba50b --- /dev/null +++ b/tools/lib/gcp-token.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Test script to access/generate secrets in Secret Manager + +# PROJECT="engineering-devops" +# SCOPE="https://www.googleapis.com/auth/cloud-platform" +# SERVICE_ACCOUNT_FILE=secret-manager.json +# SECRET=my-secret +TOKEN_TTL=3600 + + +_var2base64() { + printf "$1" | _urlencode_base64 +} + +_urlencode_base64() { + base64 | tr '/+' '_-' | tr -d '=\n' +} + +function _parse_token_uri(){ + local config_file=$1 + local token_uri=$(jq -r .token_uri ${config_file}) + echo "${token_uri}" +} + +function _generate_jwt() { + # Generate JWT token by a service account json file and scopes + local config_file=$1 + local scopes=$2 + + local now="$(date +%s)" + local expiration_time=$((${now} + ${TOKEN_TTL})) + # parse a file with credentials + local private_key=$(jq -r .private_key ${config_file}) + local client_email=$(jq -r .client_email ${config_file}) + local token_uri=$(_parse_token_uri "${config_file}") + + local claim=$(echo "{ + \"iat\": ${now}, + \"iss\": \"${client_email}\", + \"scope\": \"$scopes\", + \"aud\": \"${token_uri}\", + \"exp\":${expiration_time} + }" | jq -c) + local headers='{"typ":"JWT","alg":"RS256"}' + local body="$(_var2base64 "$headers").$(_var2base64 "$claim")" + local signature=$(openssl dgst -sha256 -sign <(echo "$private_key") <(printf "$body") | _urlencode_base64) + echo "$body.$signature" +} + +function parse_project_id(){ + # find a project_id into config file + local config_file=$1 + local project_id=$(jq -r .project_id ${config_file}) + echo "${project_id}" +} + +function get_gcp_access_token() { + # Generate an access token by a service account json file and scopes + local config_file="$1" + local scopes="$2" + local jwt=`_generate_jwt "${config_file}" "$scopes"` + local token_uri=$(_parse_token_uri "${config_file}") + local data=$(curl -s -X POST ${token_uri} \ + --data-urlencode "assertion=${jwt}" \ + --data-urlencode 'grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer' + ) + echo $data | jq -r .access_token +} +