From dd0706e8015e1a2f4631121394b0083d6d4b5b6c Mon Sep 17 00:00:00 2001 From: Sergio Ropero <42538006+sergio-ropero@users.noreply.github.com> Date: Mon, 23 Jan 2023 14:36:44 +0100 Subject: [PATCH 01/11] fix: Introduces back the `initial_waiting_seconds` option for MSSQL source. (#21348) Introduces back the `initial_waiting_seconds` option for MSSQL source. --- .../main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 14 +++++++++++++- .../source-mssql-strict-encrypt/Dockerfile | 2 +- .../src/test/resources/expected_spec.json | 9 +++++++++ .../connectors/source-mssql/Dockerfile | 2 +- .../source-mssql/src/main/resources/spec.json | 9 +++++++++ .../test-integration/resources/expected_spec.json | 9 +++++++++ docs/integrations/sources/mssql.md | 3 ++- 8 files changed, 45 insertions(+), 5 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9f92385c6812..e368e561666e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1019,7 +1019,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.4.27 + dockerImageTag: 0.4.28 documentationUrl: https://docs.airbyte.com/integrations/sources/mssql icon: mssql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index b38dcf6956ae..2b815cffb911 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8113,7 +8113,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.4.27" +- dockerImage: "airbyte/source-mssql:0.4.28" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/mssql" connectionSpecification: @@ -8272,6 +8272,18 @@ \ the \"Snapshot\" level, you must enable the snapshot isolation mode on the database." order: 2 + initial_waiting_seconds: + type: "integer" + title: "Initial Waiting Time in Seconds (Advanced)" + description: "The amount of time the connector will wait when it launches\ + \ to determine if there is new data to sync or not. Defaults to\ + \ 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about\ + \ initial waiting time." + default: 300 + min: 120 + max: 1200 + order: 3 tunnel_method: type: "object" title: "SSH Tunnel Method" diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 3ec6c69f7c27..97644ffc9f9d 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.27 +LABEL io.airbyte.version=0.4.28 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json index bbdae356015a..55ef10232e9c 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json @@ -139,6 +139,15 @@ "enum": ["Snapshot", "Read Committed"], "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", "order": 2 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 3 } } } diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index b0624aaf167e..44663ce0fe9c 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.27 +LABEL io.airbyte.version=0.4.28 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json index 35b192d2c4de..9be47072c150 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json @@ -150,6 +150,15 @@ "enum": ["Snapshot", "Read Committed"], "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", "order": 2 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 3 } } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json index 4745ff1f922e..0b94887ffc1a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json @@ -150,6 +150,15 @@ "enum": ["Snapshot", "Read Committed"], "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", "order": 2 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 3 } } } diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 44a86a62e01c..eaadcae5f40b 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -341,7 +341,8 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.27 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| 0.4.28 | 2023-01-18 | [21348](https://github.com/airbytehq/airbyte/pull/21348) | Fix error introduced in [18959](https://github.com/airbytehq/airbyte/pull/18959) in which option `initial_waiting_seconds` was removed | +| 0.4.27 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 0.4.26 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | | 0.4.25 | 2022-11-04 | [18732](https://github.com/airbytehq/airbyte/pull/18732) | Upgrade debezium version to 1.9.6 | | 0.4.24 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | From 0fbd3b284e6bab07592a529542079925f0c50aff Mon Sep 17 00:00:00 2001 From: Joey Marshment-Howell Date: Mon, 23 Jan 2023 15:13:37 +0100 Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Force=20ref?= =?UTF-8?q?resh=20of=20firebase=20token=20after=20verifying=20email=20(#21?= =?UTF-8?q?687)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/packages/cloud/lib/auth/GoogleAuthService.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts b/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts index 32c11bf9f1c6..df0f42e91754 100644 --- a/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts +++ b/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts @@ -20,6 +20,8 @@ import { signInWithPopup, GoogleAuthProvider, GithubAuthProvider, + getIdToken, + reload, } from "firebase/auth"; import { Provider } from "config"; @@ -144,7 +146,13 @@ export class GoogleAuthService { } async confirmEmailVerify(code: string): Promise { - return applyActionCode(this.auth, code); + await applyActionCode(this.auth, code); + + // Reload the user and get a fresh token with email_verified: true + if (this.auth.currentUser) { + await reload(this.auth.currentUser); + await getIdToken(this.auth.currentUser, true); + } } async signInWithEmailLink(email: string): Promise { From 79de89d4397e9561c2ebb770034536316124a627 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni <113392464+akashkulk@users.noreply.github.com> Date: Mon, 23 Jan 2023 06:33:58 -0800 Subject: [PATCH 03/11] Source Postgres : Fast query for estimate messages (#21683) * Source Postgres : Fast query for estimate messages * Update documentation * auto-bump connector version * Update strict-encrypt Dockerfile Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../source-postgres-strict-encrypt/Dockerfile | 2 +- .../connectors/source-postgres/Dockerfile | 2 +- .../source/postgres/PostgresQueryUtils.java | 2 +- .../source/postgres/PostgresSource.java | 32 ++- docs/integrations/sources/postgres.md | 241 +++++++++--------- 7 files changed, 149 insertions(+), 134 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e368e561666e..9a03ea017955 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1338,7 +1338,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.38 + dockerImageTag: 1.0.39 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 2b815cffb911..8739219dcc27 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11488,7 +11488,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.38" +- dockerImage: "airbyte/source-postgres:1.0.39" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 6e8c951d7a93..75240c9b2918 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.38 +LABEL io.airbyte.version=1.0.39 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 04a84a8f6750..2d826c4e8376 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.38 +LABEL io.airbyte.version=1.0.39 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java index 23140901e3a7..855a5d34ad2f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java @@ -26,7 +26,7 @@ public class PostgresQueryUtils { public static final String TABLE_ESTIMATE_QUERY = """ - SELECT (SELECT COUNT(*) FROM %s) AS %s, + SELECT (select reltuples::int8 as count from pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace where nspname='%s' AND relname='%s') AS %s, pg_relation_size('%s') AS %s; """; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index d2a3aab3609d..06df1a710c47 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -557,13 +557,20 @@ protected void estimateFullRefreshSyncSize(final JdbcDatabase database, final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, getQuoteString()); - final List tableEstimateResult = getFullTableEstimate(database, fullTableName); + final List tableEstimateResult = getFullTableEstimate(database, fullTableName, schemaName, tableName); if (!tableEstimateResult.isEmpty() && tableEstimateResult.get(0).has(ROW_COUNT_RESULT_COL) && tableEstimateResult.get(0).has(TOTAL_BYTES_RESULT_COL)) { final long syncRowCount = tableEstimateResult.get(0).get(ROW_COUNT_RESULT_COL).asLong(); final long syncByteCount = tableEstimateResult.get(0).get(TOTAL_BYTES_RESULT_COL).asLong(); + // The fast count query can return negative or otherwise invalid results for small tables. In this + // case, we can skip emitting an + // estimate trace altogether since the sync will likely complete quickly. + if (syncRowCount <= 0) { + return; + } + // Here, we double the bytes estimate to account for serialization. Perhaps a better way to do this // is to // read a row and Stringify it to better understand the accurate volume of data sent over the wire. @@ -588,20 +595,23 @@ protected void estimateIncrementalSyncSize(final JdbcDatabase database, final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, getQuoteString()); - final List tableEstimateResult = getFullTableEstimate(database, fullTableName); + final List tableEstimateResult = getFullTableEstimate(database, fullTableName, schemaName, tableName); final long tableRowCount = tableEstimateResult.get(0).get(ROW_COUNT_RESULT_COL).asLong(); final long tableByteCount = tableEstimateResult.get(0).get(TOTAL_BYTES_RESULT_COL).asLong(); + // The fast count query can return negative or otherwise invalid results for small tables. In this + // case, we can skip emitting an + // estimate trace altogether since the sync will likely complete quickly. + if (tableRowCount <= 0) { + return; + } + final long syncRowCount; final long syncByteCount; syncRowCount = getIncrementalTableRowCount(database, fullTableName, cursorInfo, cursorFieldType); - if (tableRowCount == 0) { - syncByteCount = 0; - } else { - syncByteCount = (tableByteCount / tableRowCount) * syncRowCount; - } + syncByteCount = (tableByteCount / tableRowCount) * syncRowCount; // Here, we double the bytes estimate to account for serialization. Perhaps a better way to do this // is to @@ -615,10 +625,14 @@ protected void estimateIncrementalSyncSize(final JdbcDatabase database, } } - private List getFullTableEstimate(final JdbcDatabase database, final String fullTableName) throws SQLException { + private List getFullTableEstimate(final JdbcDatabase database, + final String fullTableName, + final String schemaName, + final String tableName) + throws SQLException { // Construct the table estimate query. final String tableEstimateQuery = - String.format(TABLE_ESTIMATE_QUERY, fullTableName, ROW_COUNT_RESULT_COL, fullTableName, TOTAL_BYTES_RESULT_COL); + String.format(TABLE_ESTIMATE_QUERY, schemaName, tableName, ROW_COUNT_RESULT_COL, fullTableName, TOTAL_BYTES_RESULT_COL); LOGGER.debug("table estimate query: {}", tableEstimateQuery); final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.createStatement().executeQuery(tableEstimateQuery), resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 4b3ed0421585..b46e07971730 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -409,124 +409,125 @@ The root causes is that the WALs needed for the incremental sync has been remove ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1.0.38 | 2022-01-17 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | -| 1.0.37 | 2023-01-17 | [20783](https://github.com/airbytehq/airbyte/pull/20783) | Emit estimate trace messages for non-CDC mode. | -| 1.0.36 | 2023-01-11 | [21003](https://github.com/airbytehq/airbyte/pull/21003) | Handle null values for array data types in CDC mode gracefully. | -| 1.0.35 | 2023-01-04 | [20469](https://github.com/airbytehq/airbyte/pull/20469) | Introduce feature to make LSN commit behaviour configurable. | -| 1.0.34 | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378) | Improve descriptions | -| 1.0.33 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | -| 1.0.32 | 2022-12-12 | [20192](https://github.com/airbytehq/airbyte/pull/20192) | Only throw a warning if cursor column contains null values. | -| 1.0.31 | 2022-12-02 | [19889](https://github.com/airbytehq/airbyte/pull/19889) | Check before each sync and stop if an incremental sync cursor column contains a null value. | -| | 2022-12-02 | [19985](https://github.com/airbytehq/airbyte/pull/19985) | Reenable incorrectly-disabled `wal2json` CDC plugin | -| 1.0.30 | 2022-11-29 | [19024](https://github.com/airbytehq/airbyte/pull/19024) | Skip tables from schema where user do not have Usage permission during discovery. | -| 1.0.29 | 2022-11-29 | [19623](https://github.com/airbytehq/airbyte/pull/19623) | Mark PSQLException related to using replica that is configured as a hot standby server as config error. | -| 1.0.28 | 2022-11-28 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. | -| 1.0.27 | 2022-11-28 | [16990](https://github.com/airbytehq/airbyte/pull/16990) | Handle arrays data types | -| 1.0.26 | 2022-11-18 | [19551](https://github.com/airbytehq/airbyte/pull/19551) | Fixes bug with ssl modes | -| 1.0.25 | 2022-11-16 | [19004](https://github.com/airbytehq/airbyte/pull/19004) | Use Debezium heartbeats to improve CDC replication of large databases. | -| 1.0.24 | 2022-11-07 | [19291](https://github.com/airbytehq/airbyte/pull/19291) | Default timeout is reduced from 1 min to 10sec | -| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | -| 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | -| 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | -| 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | -| 1.0.19 | 2022-10-21 | [18263](https://github.com/airbytehq/airbyte/pull/18263) | Fixes bug introduced in [15833](https://github.com/airbytehq/airbyte/pull/15833) and adds better error messaging for SSH tunnel in Destinations | -| 1.0.18 | 2022-10-19 | [18087](https://github.com/airbytehq/airbyte/pull/18087) | Better error messaging for configuration errors (SSH configs, choosing an invalid cursor) | -| 1.0.17 | 2022-10-17 | [18041](https://github.com/airbytehq/airbyte/pull/18041) | Fixes bug introduced 2022-09-12 with SshTunnel, handles iterator exception properly | -| 1.0.16 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 1.0.15 | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Handle 24:00:00 value for Time column | -| 1.0.14 | 2022-10-03 | [17515](https://github.com/airbytehq/airbyte/pull/17515) | Fix an issue preventing connection using client certificate | -| 1.0.13 | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459) | Upgrade debezium version to 1.9.6 from 1.9.2 | -| 1.0.12 | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres source | -| 1.0.11 | 2022-09-26 | [17131](https://github.com/airbytehq/airbyte/pull/17131) | Allow nullable columns to be used as cursor | -| 1.0.10 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | -| 1.0.9 | 2022-09-13 | [16657](https://github.com/airbytehq/airbyte/pull/16657) | Improve CDC record queueing performance | -| 1.0.8 | 2022-09-08 | [16202](https://github.com/airbytehq/airbyte/pull/16202) | Adds error messaging factory to UI | -| 1.0.7 | 2022-08-30 | [16114](https://github.com/airbytehq/airbyte/pull/16114) | Prevent traffic going on an unsecured channel in strict-encryption version of source postgres | -| 1.0.6 | 2022-08-30 | [16138](https://github.com/airbytehq/airbyte/pull/16138) | Remove unnecessary logging | -| 1.0.5 | 2022-08-25 | [15993](https://github.com/airbytehq/airbyte/pull/15993) | Add support for connection over SSL in CDC mode | -| 1.0.4 | 2022-08-23 | [15877](https://github.com/airbytehq/airbyte/pull/15877) | Fix temporal data type bug which was causing failure in CDC mode | -| 1.0.3 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 1.0.2 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | -| 1.0.1 | 2022-08-10 | [15496](https://github.com/airbytehq/airbyte/pull/15496) | Fix state emission in incremental sync | -| | 2022-08-10 | [15481](https://github.com/airbytehq/airbyte/pull/15481) | Fix data handling from WAL logs in CDC mode | -| 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available (requires [upgrading](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to `v0.40.0-alpha`) | -| 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | -| 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | -| 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | -| 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | -| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) | -| 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | -| | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | -| 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | -| 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | -| 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. | -| 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters | -| 0.4.32 | 2022-07-07 | [14694](https://github.com/airbytehq/airbyte/pull/14694) | Force to produce LEGACY state if the use stream capable feature flag is set to false | -| 0.4.31 | 2022-07-07 | [14447](https://github.com/airbytehq/airbyte/pull/14447) | Under CDC mode, retrieve only those tables included in the publications | -| 0.4.30 | 2022-06-30 | [14251](https://github.com/airbytehq/airbyte/pull/14251) | Use more simple and comprehensive query to get selectable tables | -| 0.4.29 | 2022-06-29 | [14265](https://github.com/airbytehq/airbyte/pull/14265) | Upgrade postgresql JDBC version to 42.3.5 | -| 0.4.28 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | -| 0.4.26 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.4.25 | 2022-06-15 | [13823](https://github.com/airbytehq/airbyte/pull/13823) | Publish adaptive postgres source that enforces ssl on cloud + Debezium version upgrade to 1.9.2 from 1.4.2 | -| 0.4.24 | 2022-06-14 | [13549](https://github.com/airbytehq/airbyte/pull/13549) | Fixed truncated precision if the value of microseconds or seconds is 0 | -| 0.4.23 | 2022-06-13 | [13655](https://github.com/airbytehq/airbyte/pull/13745) | Fixed handling datetime cursors when upgrading from older versions of the connector | -| 0.4.22 | 2022-06-09 | [13655](https://github.com/airbytehq/airbyte/pull/13655) | Fixed bug with unsupported date-time datatypes during incremental sync | -| 0.4.21 | 2022-06-06 | [13435](https://github.com/airbytehq/airbyte/pull/13435) | Adjust JDBC fetch size based on max memory and max row size | -| 0.4.20 | 2022-06-02 | [13367](https://github.com/airbytehq/airbyte/pull/13367) | Added convertion hstore to json format | -| 0.4.19 | 2022-05-25 | [13166](https://github.com/airbytehq/airbyte/pull/13166) | Added timezone awareness and handle BC dates | -| 0.4.18 | 2022-05-25 | [13083](https://github.com/airbytehq/airbyte/pull/13083) | Add support for tsquey type | -| 0.4.17 | 2022-05-19 | [13016](https://github.com/airbytehq/airbyte/pull/13016) | CDC modify schema to allow null values | -| 0.4.16 | 2022-05-14 | [12840](https://github.com/airbytehq/airbyte/pull/12840) | Added custom JDBC parameters field | -| 0.4.15 | 2022-05-13 | [12834](https://github.com/airbytehq/airbyte/pull/12834) | Fix the bug that the connector returns empty catalog for Azure Postgres database | -| 0.4.14 | 2022-05-08 | [12689](https://github.com/airbytehq/airbyte/pull/12689) | Add table retrieval according to role-based `SELECT` privilege | -| 0.4.13 | 2022-05-05 | [10230](https://github.com/airbytehq/airbyte/pull/10230) | Explicitly set null value for field in json | -| 0.4.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.4.11 | 2022-04-11 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.4.10 | 2022-04-08 | [11798](https://github.com/airbytehq/airbyte/pull/11798) | Fixed roles for fetching materialized view processing | -| 0.4.8 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.4.7 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.4.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.4.5 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | -| 0.4.4 | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description | -| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | -| 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | -| 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | -| 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types | -| 0.3.17 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.3.16 | 2021-11-28 | [7995](https://github.com/airbytehq/airbyte/pull/7995) | Fixed money type with amount > 1000 | -| 0.3.15 | 2021-11-26 | [8066](https://github.com/airbytehq/airbyte/pull/8266) | Fixed the case, when Views are not listed during schema discovery | -| 0.3.14 | 2021-11-17 | [8010](https://github.com/airbytehq/airbyte/pull/8010) | Added checking of privileges before table internal discovery | -| 0.3.13 | 2021-10-26 | [7339](https://github.com/airbytehq/airbyte/pull/7339) | Support or improve support for Interval, Money, Date, various geometric data types, inventory_items, and others | -| 0.3.12 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | -| 0.3.11 | 2021-09-02 | [5742](https://github.com/airbytehq/airbyte/pull/5742) | Add SSH Tunnel support | -| 0.3.9 | 2021-08-17 | [5304](https://github.com/airbytehq/airbyte/pull/5304) | Fix CDC OOM issue | -| 0.3.8 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | -| 0.3.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.3.3 | 2021-06-08 | [3960](https://github.com/airbytehq/airbyte/pull/3960) | Add method field in specification parameters | -| 0.3.2 | 2021-05-26 | [3179](https://github.com/airbytehq/airbyte/pull/3179) | Remove `isCDC` logging | -| 0.3.1 | 2021-04-21 | [2878](https://github.com/airbytehq/airbyte/pull/2878) | Set defined cursor for CDC | -| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | -| 0.2.7 | 2021-04-16 | [2923](https://github.com/airbytehq/airbyte/pull/2923) | SSL spec as optional | -| 0.2.6 | 2021-04-16 | [2757](https://github.com/airbytehq/airbyte/pull/2757) | Support SSL connection | -| 0.2.5 | 2021-04-12 | [2859](https://github.com/airbytehq/airbyte/pull/2859) | CDC bugfix | -| 0.2.4 | 2021-04-09 | [2548](https://github.com/airbytehq/airbyte/pull/2548) | Support CDC | -| 0.2.3 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | -| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | -| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.13 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | -| 0.1.12 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | -| 0.1.11 | 2021-01-25 | [1765](https://github.com/airbytehq/airbyte/pull/1765) | Add field titles to specification | -| 0.1.10 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | -| 0.1.9 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | -| 0.1.8 | 2021-01-13 | [1588](https://github.com/airbytehq/airbyte/pull/1588) | Handle invalid numeric values in JDBC source | -| 0.1.7 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySql to use new JdbcSource | -| 0.1.6 | 2020-12-09 | [1172](https://github.com/airbytehq/airbyte/pull/1172) | Support incremental sync | -| 0.1.5 | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038) | Change JDBC sources to discover more than standard schemas | -| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.39 | 2022-01-20 | [21683](https://github.com/airbytehq/airbyte/pull/21683) | Speed up esmtimates for trace messages in non-CDC mode. | +| 1.0.38 | 2022-01-17 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| 1.0.37 | 2023-01-17 | [20783](https://github.com/airbytehq/airbyte/pull/20783) | Emit estimate trace messages for non-CDC mode. | +| 1.0.36 | 2023-01-11 | [21003](https://github.com/airbytehq/airbyte/pull/21003) | Handle null values for array data types in CDC mode gracefully. | +| 1.0.35 | 2023-01-04 | [20469](https://github.com/airbytehq/airbyte/pull/20469) | Introduce feature to make LSN commit behaviour configurable. | +| 1.0.34 | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378) | Improve descriptions | +| 1.0.33 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | +| 1.0.32 | 2022-12-12 | [20192](https://github.com/airbytehq/airbyte/pull/20192) | Only throw a warning if cursor column contains null values. | +| 1.0.31 | 2022-12-02 | [19889](https://github.com/airbytehq/airbyte/pull/19889) | Check before each sync and stop if an incremental sync cursor column contains a null value. | +| | 2022-12-02 | [19985](https://github.com/airbytehq/airbyte/pull/19985) | Reenable incorrectly-disabled `wal2json` CDC plugin | +| 1.0.30 | 2022-11-29 | [19024](https://github.com/airbytehq/airbyte/pull/19024) | Skip tables from schema where user do not have Usage permission during discovery. | +| 1.0.29 | 2022-11-29 | [19623](https://github.com/airbytehq/airbyte/pull/19623) | Mark PSQLException related to using replica that is configured as a hot standby server as config error. | +| 1.0.28 | 2022-11-28 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. | +| 1.0.27 | 2022-11-28 | [16990](https://github.com/airbytehq/airbyte/pull/16990) | Handle arrays data types | +| 1.0.26 | 2022-11-18 | [19551](https://github.com/airbytehq/airbyte/pull/19551) | Fixes bug with ssl modes | +| 1.0.25 | 2022-11-16 | [19004](https://github.com/airbytehq/airbyte/pull/19004) | Use Debezium heartbeats to improve CDC replication of large databases. | +| 1.0.24 | 2022-11-07 | [19291](https://github.com/airbytehq/airbyte/pull/19291) | Default timeout is reduced from 1 min to 10sec | +| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | +| 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | +| 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | +| 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | +| 1.0.19 | 2022-10-21 | [18263](https://github.com/airbytehq/airbyte/pull/18263) | Fixes bug introduced in [15833](https://github.com/airbytehq/airbyte/pull/15833) and adds better error messaging for SSH tunnel in Destinations | +| 1.0.18 | 2022-10-19 | [18087](https://github.com/airbytehq/airbyte/pull/18087) | Better error messaging for configuration errors (SSH configs, choosing an invalid cursor) | +| 1.0.17 | 2022-10-17 | [18041](https://github.com/airbytehq/airbyte/pull/18041) | Fixes bug introduced 2022-09-12 with SshTunnel, handles iterator exception properly | +| 1.0.16 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | +| 1.0.15 | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Handle 24:00:00 value for Time column | +| 1.0.14 | 2022-10-03 | [17515](https://github.com/airbytehq/airbyte/pull/17515) | Fix an issue preventing connection using client certificate | +| 1.0.13 | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459) | Upgrade debezium version to 1.9.6 from 1.9.2 | +| 1.0.12 | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres source | +| 1.0.11 | 2022-09-26 | [17131](https://github.com/airbytehq/airbyte/pull/17131) | Allow nullable columns to be used as cursor | +| 1.0.10 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | +| 1.0.9 | 2022-09-13 | [16657](https://github.com/airbytehq/airbyte/pull/16657) | Improve CDC record queueing performance | +| 1.0.8 | 2022-09-08 | [16202](https://github.com/airbytehq/airbyte/pull/16202) | Adds error messaging factory to UI | +| 1.0.7 | 2022-08-30 | [16114](https://github.com/airbytehq/airbyte/pull/16114) | Prevent traffic going on an unsecured channel in strict-encryption version of source postgres | +| 1.0.6 | 2022-08-30 | [16138](https://github.com/airbytehq/airbyte/pull/16138) | Remove unnecessary logging | +| 1.0.5 | 2022-08-25 | [15993](https://github.com/airbytehq/airbyte/pull/15993) | Add support for connection over SSL in CDC mode | +| 1.0.4 | 2022-08-23 | [15877](https://github.com/airbytehq/airbyte/pull/15877) | Fix temporal data type bug which was causing failure in CDC mode | +| 1.0.3 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 1.0.2 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | +| 1.0.1 | 2022-08-10 | [15496](https://github.com/airbytehq/airbyte/pull/15496) | Fix state emission in incremental sync | +| | 2022-08-10 | [15481](https://github.com/airbytehq/airbyte/pull/15481) | Fix data handling from WAL logs in CDC mode | +| 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available (requires [upgrading](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to `v0.40.0-alpha`) | +| 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | +| 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | +| 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | +| 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | +| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) | +| 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | +| | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | +| 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | +| 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | +| 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. | +| 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters | +| 0.4.32 | 2022-07-07 | [14694](https://github.com/airbytehq/airbyte/pull/14694) | Force to produce LEGACY state if the use stream capable feature flag is set to false | +| 0.4.31 | 2022-07-07 | [14447](https://github.com/airbytehq/airbyte/pull/14447) | Under CDC mode, retrieve only those tables included in the publications | +| 0.4.30 | 2022-06-30 | [14251](https://github.com/airbytehq/airbyte/pull/14251) | Use more simple and comprehensive query to get selectable tables | +| 0.4.29 | 2022-06-29 | [14265](https://github.com/airbytehq/airbyte/pull/14265) | Upgrade postgresql JDBC version to 42.3.5 | +| 0.4.28 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | +| 0.4.26 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.4.25 | 2022-06-15 | [13823](https://github.com/airbytehq/airbyte/pull/13823) | Publish adaptive postgres source that enforces ssl on cloud + Debezium version upgrade to 1.9.2 from 1.4.2 | +| 0.4.24 | 2022-06-14 | [13549](https://github.com/airbytehq/airbyte/pull/13549) | Fixed truncated precision if the value of microseconds or seconds is 0 | +| 0.4.23 | 2022-06-13 | [13655](https://github.com/airbytehq/airbyte/pull/13745) | Fixed handling datetime cursors when upgrading from older versions of the connector | +| 0.4.22 | 2022-06-09 | [13655](https://github.com/airbytehq/airbyte/pull/13655) | Fixed bug with unsupported date-time datatypes during incremental sync | +| 0.4.21 | 2022-06-06 | [13435](https://github.com/airbytehq/airbyte/pull/13435) | Adjust JDBC fetch size based on max memory and max row size | +| 0.4.20 | 2022-06-02 | [13367](https://github.com/airbytehq/airbyte/pull/13367) | Added convertion hstore to json format | +| 0.4.19 | 2022-05-25 | [13166](https://github.com/airbytehq/airbyte/pull/13166) | Added timezone awareness and handle BC dates | +| 0.4.18 | 2022-05-25 | [13083](https://github.com/airbytehq/airbyte/pull/13083) | Add support for tsquey type | +| 0.4.17 | 2022-05-19 | [13016](https://github.com/airbytehq/airbyte/pull/13016) | CDC modify schema to allow null values | +| 0.4.16 | 2022-05-14 | [12840](https://github.com/airbytehq/airbyte/pull/12840) | Added custom JDBC parameters field | +| 0.4.15 | 2022-05-13 | [12834](https://github.com/airbytehq/airbyte/pull/12834) | Fix the bug that the connector returns empty catalog for Azure Postgres database | +| 0.4.14 | 2022-05-08 | [12689](https://github.com/airbytehq/airbyte/pull/12689) | Add table retrieval according to role-based `SELECT` privilege | +| 0.4.13 | 2022-05-05 | [10230](https://github.com/airbytehq/airbyte/pull/10230) | Explicitly set null value for field in json | +| 0.4.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.4.11 | 2022-04-11 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.4.10 | 2022-04-08 | [11798](https://github.com/airbytehq/airbyte/pull/11798) | Fixed roles for fetching materialized view processing | +| 0.4.8 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.4.7 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.4.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.4.5 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | +| 0.4.4 | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description | +| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | +| 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | +| 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | +| 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types | +| 0.3.17 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.3.16 | 2021-11-28 | [7995](https://github.com/airbytehq/airbyte/pull/7995) | Fixed money type with amount > 1000 | +| 0.3.15 | 2021-11-26 | [8066](https://github.com/airbytehq/airbyte/pull/8266) | Fixed the case, when Views are not listed during schema discovery | +| 0.3.14 | 2021-11-17 | [8010](https://github.com/airbytehq/airbyte/pull/8010) | Added checking of privileges before table internal discovery | +| 0.3.13 | 2021-10-26 | [7339](https://github.com/airbytehq/airbyte/pull/7339) | Support or improve support for Interval, Money, Date, various geometric data types, inventory_items, and others | +| 0.3.12 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | +| 0.3.11 | 2021-09-02 | [5742](https://github.com/airbytehq/airbyte/pull/5742) | Add SSH Tunnel support | +| 0.3.9 | 2021-08-17 | [5304](https://github.com/airbytehq/airbyte/pull/5304) | Fix CDC OOM issue | +| 0.3.8 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| 0.3.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.3.3 | 2021-06-08 | [3960](https://github.com/airbytehq/airbyte/pull/3960) | Add method field in specification parameters | +| 0.3.2 | 2021-05-26 | [3179](https://github.com/airbytehq/airbyte/pull/3179) | Remove `isCDC` logging | +| 0.3.1 | 2021-04-21 | [2878](https://github.com/airbytehq/airbyte/pull/2878) | Set defined cursor for CDC | +| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | +| 0.2.7 | 2021-04-16 | [2923](https://github.com/airbytehq/airbyte/pull/2923) | SSL spec as optional | +| 0.2.6 | 2021-04-16 | [2757](https://github.com/airbytehq/airbyte/pull/2757) | Support SSL connection | +| 0.2.5 | 2021-04-12 | [2859](https://github.com/airbytehq/airbyte/pull/2859) | CDC bugfix | +| 0.2.4 | 2021-04-09 | [2548](https://github.com/airbytehq/airbyte/pull/2548) | Support CDC | +| 0.2.3 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | +| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | +| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.13 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | +| 0.1.12 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | +| 0.1.11 | 2021-01-25 | [1765](https://github.com/airbytehq/airbyte/pull/1765) | Add field titles to specification | +| 0.1.10 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | +| 0.1.9 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | +| 0.1.8 | 2021-01-13 | [1588](https://github.com/airbytehq/airbyte/pull/1588) | Handle invalid numeric values in JDBC source | +| 0.1.7 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySql to use new JdbcSource | +| 0.1.6 | 2020-12-09 | [1172](https://github.com/airbytehq/airbyte/pull/1172) | Support incremental sync | +| 0.1.5 | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038) | Change JDBC sources to discover more than standard schemas | +| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | From 72a7080590a13984df5bc00300cb91c59d9a8028 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Mon, 23 Jan 2023 11:47:16 -0500 Subject: [PATCH 04/11] [Low Code CDK] surface the resolved manifest in the CDK (#21703) This will be used by the connector builder server. --- .../sources/declarative/manifest_declarative_source.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py index ca010c7782ec..40fb1b6fac81 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -81,6 +81,10 @@ def __init__(self, source_config: ConnectionDefinition, debug: bool = False, con if unknown_fields: raise InvalidConnectorDefinitionException(f"Found unknown top-level fields: {unknown_fields}") + @property + def resolved_manifest(self) -> Mapping[str, Any]: + return self._new_source_config + @property def connection_checker(self) -> ConnectionChecker: check = self._new_source_config["check"] if self.construct_using_pydantic_models else self._legacy_source_config["check"] From d10dc0ccb2e0bbebe6a6b9159f067411a1fbefdc Mon Sep 17 00:00:00 2001 From: clnoll Date: Mon, 23 Jan 2023 16:49:24 +0000 Subject: [PATCH 05/11] =?UTF-8?q?=F0=9F=A4=96=20Bump=20minor=20version=20o?= =?UTF-8?q?f=20Airbyte=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index f5b3e4e22b3b..44db9b6e9a4d 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.21.0 +current_version = 0.22.0 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 2499aef31e9a..ce125c91cac2 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.22.0 +Surface the resolved manifest in the CDK + ## 0.21.0 Add AvailabilityStrategy concept and use check_availability within CheckStream diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 780ae48a81d3..ccb663e991ba 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.21.0", + version="0.22.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From b8d54717fbe801d28498a046f87c00dbb4e7f157 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 23 Jan 2023 08:50:43 -0800 Subject: [PATCH 06/11] Fix an npe on a sort fuction (#21685) * Fix an npe on a sort fuction * Format * Fix build --- .../JobCreationAndStatusUpdateActivityImpl.java | 13 ++++++++----- .../JobCreationAndStatusUpdateActivityTest.java | 10 ++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java index 295112266eca..2e321793c832 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java @@ -14,6 +14,7 @@ import static io.airbyte.persistence.job.models.AttemptStatus.FAILED; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import datadog.trace.api.Trace; import io.airbyte.commons.docker.DockerUtils; @@ -66,7 +67,6 @@ import jakarta.inject.Singleton; import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -483,10 +483,13 @@ private void failNonTerminalJobs(final UUID connectionId) { ReleaseStage.generally_available, 4); private static final Comparator RELEASE_STAGE_COMPARATOR = Comparator.comparingInt(RELEASE_STAGE_ORDER::get); - private static List orderByReleaseStageAsc(final List releaseStages) { - final List copiedList = new ArrayList<>(releaseStages); - copiedList.sort(RELEASE_STAGE_COMPARATOR); - return copiedList; + @VisibleForTesting + static List orderByReleaseStageAsc(final List releaseStages) { + // Using collector to get a mutable list + return releaseStages.stream() + .filter(stage -> stage != null) + .sorted(RELEASE_STAGE_COMPARATOR) + .toList(); } /** diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java index ccdb8447fd6c..0efcb27e5c6d 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java @@ -33,6 +33,7 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; import io.airbyte.persistence.job.JobCreator; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; @@ -535,4 +536,13 @@ void ensureCleanJobState() throws IOException { } + @Test + void testReleaseStageOrdering() { + final List input = List.of(ReleaseStage.alpha, ReleaseStage.custom, ReleaseStage.beta, ReleaseStage.generally_available); + final List expected = List.of(ReleaseStage.custom, ReleaseStage.alpha, ReleaseStage.beta, ReleaseStage.generally_available); + + Assertions.assertThat(JobCreationAndStatusUpdateActivityImpl.orderByReleaseStageAsc(input)) + .containsExactlyElementsOf(expected); + } + } From 6595409098a50b64aaabf958c9951361c5305d2e Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Mon, 23 Jan 2023 11:56:33 -0500 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=A8=20=20Update?= =?UTF-8?q?=20page=20bottom=20margins=20to=20be=20consistent=20(#21673)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix bottom of Sources and destinations settings page * Remove unused components * Remove H3 from titles * Fix layouts to have consistent page bottom margin * Set default spacing to 20px / xl * Remove extraneous padding from pages and let containers handle it * Check if running cloud app to add extra bottom spacing * Fix layout issues with connection Edit controls, move line rendering to the transformation cards * Set "saving" state in create controls to use the create button instead of having a different UI * Fix padding in DbtTransformationsCard --- .../FormPageContent.module.scss | 6 +- .../ConnectorBlocks/FormPageContent.tsx | 11 +++- .../src/components/base/Titles/Titles.tsx | 6 -- .../src/components/base/Titles/index.tsx | 2 +- .../MainPageWithScroll.module.scss | 7 ++- .../MainPageWithScroll/MainPageWithScroll.tsx | 5 +- .../ConnectionEditFormCard.module.scss | 9 ++- .../ConnectionEditFormCard.tsx | 5 +- .../ConnectionForm/CreateControls.module.scss | 5 ++ .../ConnectionForm/CreateControls.tsx | 40 ++----------- .../ConnectionForm/EditControls.module.scss | 28 +-------- .../ConnectionForm/EditControls.tsx | 47 +++++++-------- .../CollapsibleCard.module.scss | 3 - .../ui/CollapsibleCard/CollapsibleCard.tsx | 4 -- .../SettingsPage/SettingsPage.module.scss | 1 - .../components/ConnectorsView.module.scss | 3 - .../components/ConnectorsView.tsx | 59 +++++++++++-------- .../components/PageComponents.tsx | 20 +------ .../ConnectionReplicationPage.module.scss | 2 - .../ConnectionStatusPage.module.scss | 4 -- .../ConnectionStatusPage.tsx | 1 - .../ConnectionTransformationPage.module.scss | 12 +++- .../ConnectionTransformationPage.tsx | 5 +- .../CustomTransformationsCard.tsx | 1 - .../DbtCloudTransformationsCard.module.scss | 2 +- airbyte-webapp/src/scss/_variables.scss | 2 +- .../ConnectorDocumentationLayout.module.scss | 2 +- 27 files changed, 119 insertions(+), 173 deletions(-) create mode 100644 airbyte-webapp/src/components/connection/ConnectionForm/CreateControls.module.scss delete mode 100644 airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.module.scss delete mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.module.scss diff --git a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss index a3166da8dda7..524972b7ce19 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss +++ b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss @@ -2,7 +2,11 @@ .container { margin: 13px auto 0; - padding-bottom: variables.$spacing-page-bottom; + padding-bottom: variables.$spacing-xl; + + &.cloud { + padding-bottom: variables.$spacing-page-bottom-cloud; + } &:not(.big) { width: 80%; diff --git a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx index 64af7f776f97..1e1c4a051ec5 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx +++ b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx @@ -1,6 +1,8 @@ import classNames from "classnames"; import { PropsWithChildren } from "react"; +import { isCloudApp } from "utils/app"; + import styles from "./FormPageContent.module.scss"; interface FormPageContentProps { @@ -8,7 +10,14 @@ interface FormPageContentProps { } const FormPageContent: React.FC> = ({ big, children }) => ( -
{children}
+
+ {children} +
); export default FormPageContent; diff --git a/airbyte-webapp/src/components/base/Titles/Titles.tsx b/airbyte-webapp/src/components/base/Titles/Titles.tsx index 9d2a5ee0fd9d..50630f7ce478 100644 --- a/airbyte-webapp/src/components/base/Titles/Titles.tsx +++ b/airbyte-webapp/src/components/base/Titles/Titles.tsx @@ -19,12 +19,6 @@ const H1 = styled.h1` margin: 0; `; -/** @deprecated Use `` */ -export const H3 = styled(H1).attrs({ as: "h3" })` - font-size: 20px; - line-height: 24px; -`; - /** @deprecated Use `` */ export const H5 = styled(H1).attrs({ as: "h5" })` font-size: ${({ theme }) => theme.h5?.fontSize || "16px"}; diff --git a/airbyte-webapp/src/components/base/Titles/index.tsx b/airbyte-webapp/src/components/base/Titles/index.tsx index 502ba2ea7036..c6f9a9079c5f 100644 --- a/airbyte-webapp/src/components/base/Titles/index.tsx +++ b/airbyte-webapp/src/components/base/Titles/index.tsx @@ -1 +1 @@ -export { H3, H5 } from "./Titles"; +export { H5 } from "./Titles"; diff --git a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss index 22c55a7f45fb..185526591e75 100644 --- a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss +++ b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss @@ -9,6 +9,7 @@ } .contentContainer { + flex: 1; max-width: 100%; overflow-x: auto; padding-top: variables.$spacing-lg; @@ -17,6 +18,10 @@ .content { overflow-y: auto; height: 100%; - padding: 0 variables.$spacing-xl variables.$spacing-page-bottom; + padding: 0 variables.$spacing-xl variables.$spacing-xl; min-width: variables.$main-page-content-min-width; + + &.cloud { + padding-bottom: variables.$spacing-page-bottom-cloud; + } } diff --git a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx index 047e1e14dbe1..71091c50b3cb 100644 --- a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx +++ b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx @@ -1,5 +1,8 @@ +import classNames from "classnames"; import React from "react"; +import { isCloudApp } from "utils/app"; + import styles from "./MainPageWithScroll.module.scss"; /** @@ -20,7 +23,7 @@ export const MainPageWithScroll: React.FC = ({ headTitl {pageTitle}
-
{children}
+
{children}
); diff --git a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss index ce45b41bb4f7..343ffae519be 100644 --- a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss +++ b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss @@ -1,5 +1,12 @@ +@use "scss/colors"; @use "scss/variables"; .formCard { - padding: 22px 27px variables.$spacing-xl 24px; + padding: variables.$spacing-xl; +} + +.editControls { + border-top: variables.$border-thin solid colors.$grey-100; + margin: 0 -#{variables.$spacing-xl}; + padding: variables.$spacing-xl variables.$spacing-xl 0; } diff --git a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx index a0be5283d333..3533cafa8a4a 100644 --- a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx @@ -13,7 +13,6 @@ import EditControls from "../ConnectionForm/EditControls"; import styles from "./ConnectionEditFormCard.module.scss"; interface FormCardProps extends CollapsibleCardProps { - bottomSeparator?: boolean; form: FormikConfig; submitDisabled?: boolean; } @@ -21,7 +20,6 @@ interface FormCardProps extends CollapsibleCardProps { export const ConnectionEditFormCard = ({ children, form, - bottomSeparator = true, submitDisabled, ...props }: React.PropsWithChildren>) => { @@ -45,10 +43,9 @@ export const ConnectionEditFormCard = ({
{children} -
+
{mode !== "readonly" && ( theme.darkPrimaryColor}; - justify-content: center; -`; - -const Loader = styled.div` - margin-right: 10px; -`; - const Success = styled(StatusIcon)` width: 26px; min-width: 26px; @@ -61,19 +44,8 @@ const ErrorText = styled.div` `; const CreateControls: React.FC = ({ isSubmitting, errorMessage, isValid }) => { - if (isSubmitting) { - return ( - - - - - - - ); - } - return ( - + {errorMessage ? ( @@ -86,11 +58,11 @@ const CreateControls: React.FC = ({ isSubmitting, errorMess
)}
-
- + ); }; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss index 2afd4f0946a3..1610ff1d8d2c 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss +++ b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss @@ -1,29 +1,5 @@ -@use "scss/colors"; @use "scss/variables"; -.content { - display: flex; - justify-content: flex-end; - align-items: center; - flex-direction: row; - margin-top: variables.$spacing-lg; - gap: variables.$spacing-lg; - padding: variables.$spacing-sm; -} - -.controlButton { - margin-left: variables.$spacing-md; -} - -// currently only implemented on transformation view form card, margins are specific to that implementation -// todo: standardize the margin sizes here -.line { - min-width: 100%; - height: variables.$border-thin; - background: colors.$grey-100; - margin: variables.$spacing-lg -27px 0 -24px; -} - -.buttonsContainer { - display: flex; +.container { + margin-top: variables.$spacing-md; } diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx index fc78785d6458..6fa6fe33cd1b 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx @@ -2,6 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; import styles from "./EditControls.module.scss"; import { ResponseMessage } from "./ResponseMessage"; @@ -14,7 +15,6 @@ interface EditControlProps { successMessage?: React.ReactNode; errorMessage?: React.ReactNode; enableControls?: boolean; - withLine?: boolean; } const EditControls: React.FC = ({ @@ -25,33 +25,28 @@ const EditControls: React.FC = ({ successMessage, errorMessage, enableControls, - withLine, }) => { return ( - <> - {withLine &&
} -
- -
- - -
-
- + + + + + + + ); }; diff --git a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.module.scss b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.module.scss deleted file mode 100644 index 87ff85fde155..000000000000 --- a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.collapsibleCard { - margin-bottom: 10px; -} diff --git a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx index 4b0dfdaeec9d..52b12607e2bd 100644 --- a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx +++ b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx @@ -1,14 +1,11 @@ import { faChevronRight } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import classNames from "classnames"; import React from "react"; import { useToggle } from "react-use"; import styled from "styled-components"; import { Card } from "components/ui/Card"; -import styles from "./CollapsibleCard.module.scss"; - const CardHeader = styled.div` display: flex; justify-content: space-between; @@ -40,7 +37,6 @@ export const CollapsibleCard: React.FC = ({ return ( {title} diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss index a756f2ddb779..997a2399432e 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss @@ -3,7 +3,6 @@ .content { display: flex; flex-direction: row; - padding-bottom: variables.$spacing-lg; } .mainView { diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.module.scss deleted file mode 100644 index 17d92ee1b1e6..000000000000 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.buttonsContainer { - display: flex; -} diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx index 21030a199977..fa40c8a3aedb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx @@ -3,6 +3,8 @@ import { FormattedMessage } from "react-intl"; import { CellProps } from "react-table"; import { HeadTitle } from "components/common/HeadTitle"; +import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { Table } from "components/ui/Table"; import { Connector, ConnectorDefinition } from "core/domain/connector"; @@ -12,10 +14,9 @@ import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import ConnectorCell from "./ConnectorCell"; -import styles from "./ConnectorsView.module.scss"; import CreateConnector from "./CreateConnector"; import ImageCell from "./ImageCell"; -import { Block, FormContentTitle, Title } from "./PageComponents"; +import { FormContentTitle } from "./PageComponents"; import UpgradeAllButton from "./UpgradeAllButton"; import VersionCell from "./VersionCell"; @@ -125,7 +126,7 @@ const ConnectorsView: React.FC = ({ const renderHeaderControls = (section: "used" | "available") => ((section === "used" && usedConnectorsDefinitions.length > 0) || (section === "available" && usedConnectorsDefinitions.length === 0)) && ( -
+ {allowUploadCustomImage && } {(hasNewConnectorVersion || isUpdateSuccess) && allowUpdateConnectors && ( = ({ onUpdate={onUpdate} /> )} -
+ ); return ( @@ -143,31 +144,37 @@ const ConnectorsView: React.FC = ({ - {usedConnectorsDefinitions.length > 0 && ( - - - <FormattedMessage id={type === "sources" ? "admin.manageSource" : "admin.manageDestination"} /> - {renderHeaderControls("used")} - + + {usedConnectorsDefinitions.length > 0 && ( + + + + + + {renderHeaderControls("used")} + + + + )} + + + + + + + {renderHeaderControls("available")} +
- - )} - - - - <FormattedMessage id={type === "sources" ? "admin.availableSource" : "admin.availableDestinations"} /> - {renderHeaderControls("available")} - -
- + + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx index 31a9eaff246d..ef7827175dd4 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx @@ -1,27 +1,11 @@ import styled from "styled-components"; -import { H5 } from "components/base/Titles"; - -const Title = styled(H5)` - color: ${({ theme }) => theme.darkPrimaryColor}; - margin-bottom: 19px; - display: flex; - justify-content: space-between; - align-items: center; -`; - -const Block = styled.div` - margin-bottom: 56px; -`; - -const FormContent = styled.div` +export const FormContent = styled.div` width: 253px; margin: -10px 0 -10px 200px; position: relative; `; -const FormContentTitle = styled(FormContent)` +export const FormContentTitle = styled(FormContent)` margin: 0 0 0 200px; `; - -export { Title, Block, FormContent, FormContentTitle }; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss index fc965f61fed9..1bba26e32786 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss @@ -1,8 +1,6 @@ @use "scss/variables"; .content { - padding-bottom: variables.$spacing-md; - > form { display: flex; flex-direction: column; diff --git a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss index 48419e552f16..8badacf0e855 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss @@ -31,10 +31,6 @@ } } -.contentCard { - margin-bottom: variables.$spacing-xl; -} - .footer { width: 100%; display: flex; diff --git a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx index 29bcdb7986bb..c5f87cbf6a4b 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx @@ -159,7 +159,6 @@ export const ConnectionStatusPage: React.FC = () => { return ( <> diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss index 6903e64a247b..681bc32cc405 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss @@ -1,7 +1,17 @@ +@use "scss/variables"; + .content { max-width: 1073px; margin: 0 auto; - padding-bottom: 10px; + + & > fieldset { + margin: 0; + padding: 0; + border: 0; + display: flex; + flex-direction: column; + gap: variables.$spacing-lg; + } } .customCard { diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx index 17bd026c7c27..40c9ee19e2b9 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx @@ -61,10 +61,7 @@ export const ConnectionTransformationPage: React.FC = () => { return (
-
+
{supportsNormalization && } {supportsDbt && } {supportsCloudDbtIntegration && } diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx index 0b8a0b8455f3..88a58210773c 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx @@ -28,7 +28,6 @@ export const CustomTransformationsCard: React.FC<{ title={} collapsible - bottomSeparator form={{ initialValues, enableReinitialize: true, diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss index 09620779260b..9d9a85316dbc 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss @@ -5,7 +5,7 @@ display: flex; flex-direction: column; align-items: center; - padding: 25px 25px 22px; + padding: variables.$spacing-xl; background-color: colors.$grey-50; } diff --git a/airbyte-webapp/src/scss/_variables.scss b/airbyte-webapp/src/scss/_variables.scss index 5f5d103e38b0..19c967957ab5 100644 --- a/airbyte-webapp/src/scss/_variables.scss +++ b/airbyte-webapp/src/scss/_variables.scss @@ -26,7 +26,7 @@ $spacing-md: 10px; $spacing-lg: 15px; $spacing-xl: 20px; $spacing-2xl: 40px; -$spacing-page-bottom: 150px; +$spacing-page-bottom-cloud: 88px; $main-page-content-min-width: 960px; $width-size-menu: 93px; diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss index 626cae0e8051..41c28fbedb93 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss +++ b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss @@ -3,7 +3,7 @@ .leftPanel { > *:last-child { - padding-bottom: variables.$spacing-page-bottom; + padding-bottom: variables.$spacing-page-bottom-cloud; } } From db37aadf67d21d0a8f93256fbcd5003e00f70242 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Mon, 23 Jan 2023 09:12:27 -0800 Subject: [PATCH 08/11] Improve Authentication Error handling (#21087) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor checks * Add Auth Error Checks * rebase from master * some weird versioning issue * 🪟🎉 Connector builder: Substream slicer and cartesian slicer (#20861) * improve some types * improve further * clean up a bit more * refactor loading state * move loading state up * remove isLoading references * remove unused props and make fetch connector error work * remove special component for name * remove top level state for unifinished flows * start removing uiwidget * Update airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss Co-authored-by: Tim Roes * remove undefined option for selected id * remove unused prop * fix types * remove uiwidget state * clean up * adjust comment * handle errors in a nice way * do not respect default on oneOf fields * rename to formblock * reduce re-renders * pass error to secure inputs * simplify and improve styling * align top * code review * remove comment * review comments * rename file * be strict about boolean values * add example * track form error in error boundary * review comments * handle unexpected cases better * enrich error with connector id * 🪟🎉 Add copy stream button (#20577) * add copy stream button * review comments * rename prop * 🪟🎉 Connector builder: Integrate connector form for test input (#20385) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * fix small stuff * add warning label * review comments * adjust translation Co-authored-by: lmossman * use request_body_json instead of request_body_data * :window: :art: Move `Add` button into the line of Connector Builder key value list fields (#20699) * move add button into line * add stories for empty with control, and content + control * change button name to Control * 🪟🎉 Connector builder: Allow defining inputs (#20431) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * handle stored form values that don't contain new fields properly * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * 🪟🎉 Connector builder authentication (#20645) * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * fix keys * 🪟🎉 Connector builder: Session token and oauth authentication (#20712) * session token and oauth authentication * fill in session token variable * typos * make sure validation error does not go away * 🪟🎉 Connector builder: Always validate inputs form (#20664) * validate user input outside of form * review comments Co-authored-by: lmossman Co-authored-by: lmossman * fix merge conflict with dropdown prop being renamed to control * [Connector Builder] Add paginator (#20698) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly Co-authored-by: Joe Reuter * [Connector Builder] Add stream slicer (#20748) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly * save stream slicer progress * finish stream slicer * fix stream slicer fields and validation Co-authored-by: Joe Reuter * debounce form builder values update to reduce load * 🪟🔧 Connector builder: use new lowcode manifest (#20715) * use new manifest yaml * Update airbyte-webapp/src/components/connectorBuilder/types.ts Co-authored-by: Lake Mossman * use updated manifest types Co-authored-by: Lake Mossman * add substream slicer * add substream and cartesian slicer * debounce validation as well * akways show stream test button in error state if there are errors * fix type of oauth input * add validation schema for add stream form * validate all views on test click * add type to prevent console warning * do not allow path for substream slicer request option * do not show request option for substream slicer * rewrite stream slice field tooltip Co-authored-by: Tim Roes Co-authored-by: lmossman * 🪟🔧 Connector builder: Performance improvements (#20620) * improve some types * improve further * clean up a bit more * refactor loading state * move loading state up * remove isLoading references * remove unused props and make fetch connector error work * remove special component for name * remove top level state for unifinished flows * start removing uiwidget * Update airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss Co-authored-by: Tim Roes * remove undefined option for selected id * remove unused prop * fix types * remove uiwidget state * clean up * adjust comment * handle errors in a nice way * do not respect default on oneOf fields * rename to formblock * reduce re-renders * pass error to secure inputs * simplify and improve styling * align top * code review * remove comment * review comments * rename file * be strict about boolean values * add example * track form error in error boundary * review comments * handle unexpected cases better * speed up some bits * more changes * enrich error with connector id * 🪟🎉 Add copy stream button (#20577) * add copy stream button * review comments * rename prop * 🪟🎉 Connector builder: Integrate connector form for test input (#20385) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * fix small stuff * add warning label * review comments * adjust translation Co-authored-by: lmossman * use request_body_json instead of request_body_data * :window: :art: Move `Add` button into the line of Connector Builder key value list fields (#20699) * move add button into line * add stories for empty with control, and content + control * change button name to Control * 🪟🎉 Connector builder: Allow defining inputs (#20431) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * handle stored form values that don't contain new fields properly * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * 🪟🎉 Connector builder authentication (#20645) * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * fix keys * 🪟🎉 Connector builder: Session token and oauth authentication (#20712) * session token and oauth authentication * fill in session token variable * typos * make sure validation error does not go away * 🪟🎉 Connector builder: Always validate inputs form (#20664) * validate user input outside of form * review comments Co-authored-by: lmossman Co-authored-by: lmossman * fix merge conflict with dropdown prop being renamed to control * [Connector Builder] Add paginator (#20698) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly Co-authored-by: Joe Reuter * [Connector Builder] Add stream slicer (#20748) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly * save stream slicer progress * finish stream slicer * fix stream slicer fields and validation Co-authored-by: Joe Reuter * debounce form builder values update to reduce load * 🪟🔧 Connector builder: use new lowcode manifest (#20715) * use new manifest yaml * Update airbyte-webapp/src/components/connectorBuilder/types.ts Co-authored-by: Lake Mossman * use updated manifest types Co-authored-by: Lake Mossman * debounce validation as well * akways show stream test button in error state if there are errors * fix type of oauth input * review comments * fix more * add validation schema for add stream form * validate all views on test click * add type to prevent console warning * review comment * make sure testing state and form state stay consistent * improve builder errors * remove test state from streamconfig view * remove console log * remove unnecessary positive index check Co-authored-by: Tim Roes Co-authored-by: lmossman * Remove workspace helper from fetchConfigActivity (#21048) * Remove workspace helper and replace with workspaceApi * Publish new version of destination-redshift (#21083) * Update changelog * auto-bump connector version Co-authored-by: Octavia Squidington III * Progress Bar Read APIs (#20937) Follow up PR to #20787 . Make stats available to the read apis so these are available to the webapp. After this, all that is left is writing these stats as the job progresses. Add the required logic in JobHistoryHandler.java. Took the chance to also rename our internal Attempt models field from id to attemptNumber to better reflect that the field stores not the row's database id, but the job's attempt number. Most of the files changes here are due to that rename. * Rm temporal version (#21045) * Rm temporal version * Remove temporal version * Update the replayed workflow * Format * Fix pmd * 🪟🎉 Connector builder: Available inputs dropdown (#20983) * improve some types * improve further * clean up a bit more * refactor loading state * move loading state up * remove isLoading references * remove unused props and make fetch connector error work * remove special component for name * remove top level state for unifinished flows * start removing uiwidget * Update airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss Co-authored-by: Tim Roes * remove undefined option for selected id * remove unused prop * fix types * remove uiwidget state * clean up * adjust comment * handle errors in a nice way * do not respect default on oneOf fields * rename to formblock * reduce re-renders * pass error to secure inputs * simplify and improve styling * align top * code review * remove comment * review comments * rename file * be strict about boolean values * add example * track form error in error boundary * review comments * handle unexpected cases better * enrich error with connector id * 🪟🎉 Add copy stream button (#20577) * add copy stream button * review comments * rename prop * 🪟🎉 Connector builder: Integrate connector form for test input (#20385) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * fix small stuff * add warning label * review comments * adjust translation Co-authored-by: lmossman * use request_body_json instead of request_body_data * :window: :art: Move `Add` button into the line of Connector Builder key value list fields (#20699) * move add button into line * add stories for empty with control, and content + control * change button name to Control * 🪟🎉 Connector builder: Allow defining inputs (#20431) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * handle stored form values that don't contain new fields properly * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * 🪟🎉 Connector builder authentication (#20645) * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * fix keys * 🪟🎉 Connector builder: Session token and oauth authentication (#20712) * session token and oauth authentication * fill in session token variable * typos * make sure validation error does not go away * 🪟🎉 Connector builder: Always validate inputs form (#20664) * validate user input outside of form * review comments Co-authored-by: lmossman Co-authored-by: lmossman * fix merge conflict with dropdown prop being renamed to control * [Connector Builder] Add paginator (#20698) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly Co-authored-by: Joe Reuter * [Connector Builder] Add stream slicer (#20748) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly * save stream slicer progress * finish stream slicer * fix stream slicer fields and validation Co-authored-by: Joe Reuter * debounce form builder values update to reduce load * 🪟🔧 Connector builder: use new lowcode manifest (#20715) * use new manifest yaml * Update airbyte-webapp/src/components/connectorBuilder/types.ts Co-authored-by: Lake Mossman * use updated manifest types Co-authored-by: Lake Mossman * debounce validation as well * akways show stream test button in error state if there are errors * fix type of oauth input * available inputs dropdown * add validation schema for add stream form * validate all views on test click * add type to prevent console warning * improve styling * make sure padding is set correctly * make sure focus is set right * comment * use correct state hook * add tooltip to user input button and fix wording for new user input option Co-authored-by: Tim Roes Co-authored-by: lmossman * Fixed appstore docs link (#21098) * fill in all default values on switch (#21059) * 🐛Source Freshdesk: Fix schema types (#20416) * Fix schema types * Updated version * Updated acceptance tests * auto-bump connector version Co-authored-by: Octavia Squidington III * CDK: Add schema inferrer class (#20941) * fix stuff * Update schema_inferrer.py * Update schema_inferrer.py * bump version * review comments * code style * fix formatting * improve tests * Source Facebook Marketing: Docs update (#21105) * Source Amazon Ads: fix bug with handling: "Report date is too far in the past." (partial revert of #20662) (#21082) * Revert "Source Amazon Ads: fix bug with handling: "Report date is too far in the past." (#20662)" This reverts commit ec995959f7af3574cd7a6a5e63d2c2aaf3118183. * fix Signed-off-by: Sergey Chvalyuk * revert master Signed-off-by: Sergey Chvalyuk * revert to master Signed-off-by: Sergey Chvalyuk * bump 0.1.27 Signed-off-by: Sergey Chvalyuk * fix Signed-off-by: Sergey Chvalyuk * amazon-ads.md updated Signed-off-by: Sergey Chvalyuk * amazon-ads.md updated Signed-off-by: Sergey Chvalyuk * auto-bump connector version Signed-off-by: Sergey Chvalyuk Co-authored-by: Augustin Co-authored-by: Octavia Squidington III Co-authored-by: Topher Lubaway * Revert "Progress Bar Read APIs (#20937)" (#21115) Breaks when there is no config present https://github.com/airbytehq/airbyte/issues/21112 This reverts commit 3a2b0405c425562ba5fa8bfe3f5a4646c165591a. * Remove unneeded margin top and bottom (#21111) * feat(Platform): update actor configuration when receiving control messages from connectors during sync (#19811) * track latest config message * pass new config as part of outputs * persist new config * persist config as the messages come through, dont set output * clean up old implementation * accept control messages for destinations * get api client from micronaut * mask instance-wide oauth params when updating configs * defaultreplicationworker tests * formatting * tests for source/destination handlers * rm todo * refactor test a bit to fix pmd * fix pmd * fix test * add PersistConfigHelperTest * update message tracker comment * fix pmd * format * move ApiClientBeanFactory to commons-worker, use in container-orchestrator * pull out config updating to separate methods * add jitter * rename PersistConfigHelper -> UpdateConnectorConfigHelper, docs * fix exception type * fmt * move message type check into runnable * formatting * pass api client env vars to container orchestrator * pass micronaut envs to container orchestrator * print stacktrace for debugging * different api host for container orchestrator * fix default env var * format * fix errors after merge * set source and destination actor id as part of the sync input * fix: get destination definition * fix null ptr * remove "actor" from naming * fix missing change from rename * revert ContainerOrchestratorConfigBeanFactory changes * inject sourceapi/destinationapi directly rather than airbyteapiclient * UpdateConnectorConfigHelper -> ConnectorConfigUpdater * rm log * fix test * dont fail on config update error * pass id, not full config to runnables/accept control message * add new config required for api client * add test file * fix test compatibility * mount data plane credentials secret to container orchestrator (#20724) * mount data plane credentials secret to container orchestrator * rm copy-pasta * properly handle empty strings * set env vars like before * use the right config vars * 🐛Source Looker: Fix schema transformation issue (#20182) * Fix schema transformation issue * Updated PR number * Added unittest * Add a test case that uses the recursion * Unhide from cloud * Bumed seed version * [ISSUE #20322] add datetime_granularity logic to DatetimeStreamSlicer… (#20717) * [ISSUE #20322] add datetime_granularity logic to DatetimeStreamSlicer and migrate duration to ISO8601 * [ISSUE #20322] fix tests * [ISSUE #20322] code review based on clnoll's comments and fixed tests * [ISSUE #20322] fix flake8 error * [ISSUE #20322] fix source tests * [ISSUE #20322] fixing yet another error in source * [ISSUE #20322] code review * [ISSUE #20322] adding new sources using datetime slicer * [ISSUE #20322] fixing source-datascope and increasing version * [ISSUE #20322] regenerate component schema * [ISSUE #20322] fixing source-datascope * [ISSUE #20322] extra field error * Source Slack: update schema; data from openapi spec (#20767) * Source Slack: update schema; data from openapi spec * Source Slack: Docs update * Source Slack: Update schema * Source Slack: Update schema (timestamp returned in POSIX time) * Source Slack: Disable backward compatibility test for discovery * auto-bump connector version Co-authored-by: Octavia Squidington III * make sure name is rendered in first position (#21101) * Source Github: Raise Error if no organizations or repos are available (#21084) * Source Github: Raise Error if no organizations or repos are available * Source GitHub: Update docs; bump version * Source GitHub: Add test * Source GitHub: Exclude empty streams * auto-bump connector version Co-authored-by: Octavia Squidington III * Convert the server to micronaut (#19194) * Extract Operation API * Extract scheduler API * Format * extract source api * Extract source definition api * Add path * Extract State API * extract webbackend api * extract webbackend api * extract workspace api * Extract source definition specification api * Remove configuration API * tmp * Checkstyle * tmp * tmp * Inject but don't resolve Bean * tmp * Tmp * fix build * TMP * Tmp * Clean up * better thread pool * Change port to 8080 * Fix port * Rm unused * Cors filter * Format * rename * Tmp * Config based * Rm health controller ref * tmp * Pool size * Mock healthcheck * Revert "Mock healthcheck" This reverts commit 466677636bda96bed7bbfd354a59abc38ff1e315. * Revert "Revert "Mock healthcheck"" This reverts commit 267094ae149f3c077bdba6f71646d99655ce81a0. * Restore health check * Tmp * format * Rm deprecated * Fix PMD * Tmp * Fix proxy test * Remove useless annotation * set auto commit as false * Clean up and PR comments * Bmoric/convert attempt micronaut (#19847) * tmp * Fix build * tmp * Tmp * tmp * tmp * Tmp * tmp * tmp * Clean up * Comments and banner * Non related files * rm tmp * Fix build * Format * Hit the micronaut server directly * micronaut OperationApiController (#20270) * micronaut OperationApiController * pass micronaut client to OperationApi * Bmoric/convert connection micronaut (#20211) * tmp * Fix build * tmp * Tmp * tmp * tmp * Tmp * tmp * tmp * Clean up * tmp * Convert Connection Api Controller * PR Comments * convert openapiapicontroller to micronaut (#20258) * convert openapiapicontroller to micronaut * merge health/openapi locations into one entry * Fix build * Format * Remove media type * Format Co-authored-by: Cole Snodgrass * Bmoric/convert destination controller micronaut (#20269) * tmp * Fix build * tmp * Tmp * tmp * tmp * Tmp * tmp * tmp * Clean up * tmp * Convert Connection Api Controller * Tmp * PR Comments * convert openapiapicontroller to micronaut (#20258) * convert openapiapicontroller to micronaut * merge health/openapi locations into one entry * Fix bean * Add JsonSchemaValidator as a Bean * Fix build * Format * Format * Test fix * Pr comments * Remove media type * Format * Remove media type * Format * format * Add missing airbyte api client Co-authored-by: Cole Snodgrass * Bmoric/convert destination definition controller micronaut (#20277) * tmp * Fix build * tmp * Tmp * tmp * tmp * Tmp * tmp * tmp * Clean up * tmp * Convert Connection Api Controller * Tmp * PR Comments * convert openapiapicontroller to micronaut (#20258) * convert openapiapicontroller to micronaut * merge health/openapi locations into one entry * Fix bean * Add JsonSchemaValidator as a Bean * Fix build * Format * Format * Test fix * Pr comments * Remove media type * Format * Remove media type * Format * Remove media type * Format * api client * missing annotation * format Co-authored-by: Cole Snodgrass * convert StateApiController to Micronaut (#20329) * convert to micronaut * nginx updates * format * Move dest oauth to micronaut (#20318) * tmp * Fix build * tmp * Tmp * tmp * tmp * Tmp * tmp * tmp * Clean up * tmp * Convert Connection Api Controller * PR Comments * convert openapiapicontroller to micronaut (#20258) * convert openapiapicontroller to micronaut * merge health/openapi locations into one entry * Fix build * Format * Remove media type * Format * Move dest oauth to micronaut * Pr comments * format Co-authored-by: Cole Snodgrass * Bmoric/convert source micronaut (#20334) * tmp * Fix build * tmp * Tmp * tmp * tmp * Tmp * tmp * tmp * Clean up * tmp * Convert Connection Api Controller * PR Comments * convert openapiapicontroller to micronaut (#20258) * convert openapiapicontroller to micronaut * merge health/openapi locations into one entry * Fix build * Format * Remove media type * Format * Tmp * tmp * Build * missing bean * format Co-authored-by: Cole Snodgrass * Migrate to micronaut (#20339) * Migrate source to micronaut * convert SchedulerApiController to Micronaut (#20337) * wip; SchedulerApiController * remove @Named * remove @Singleton * add back todo message * Bmoric/convert source definition micronaut (#20338) * tmp * Fix build * tmp * Tmp * tmp * tmp * Tmp * tmp * tmp * Clean up * tmp * Convert Connection Api Controller * PR Comments * convert openapiapicontroller to micronaut (#20258) * convert openapiapicontroller to micronaut * merge health/openapi locations into one entry * Fix build * Format * Remove media type * Format * Tmp * tmp * Build * missing bean * Tmp * Add Beans * fix Bean * Add passthrough * Clean up * Missing path * FIx typo * Fix conflicts * for mat Co-authored-by: Cole Snodgrass * update SourceOauthApiController to Micronaut (#20386) * convert SourceOauthApiController to Micronaut * remove SourceOauthApi reference * convert WorkspaceApiController to micronaut (#20214) * wip; broken * convert WorkspaceApiController to micronaut * remove test controller * format * format * add @Body to SourceOauthApiController * consolidate nginx settings * remove unnecessary factories * Bmoric/convert jobs micronaut (#20382) * Convert jobs to micronaut * Nit * Format * Bmoric/convert source definition specification micronaut (#20379) * Migrate source definition specifications to micronaut * Format * Format * convert database assert call to Micronaut (#20406) * remove dupe config section; add DatabaseEventListener * move eventlistner to correct package; update implementation * convert NotificationsApiController to Micronaut (#20396) * convert NotificationsApiController to Micronaut * format * Migrate logs to micronaut (#20400) * Bmoric/convert webbackend micronaut (#20403) * Convert jobs to micronaut * Nit * Format * Migrate the webbackend to micronaut * Add missing Bean * Cleanup (#20459) * Cleanup * More cleanup * Disable in order to test cloud * Restore missing files * Fix test * Format and fix pmd * Add transactional * Fix version * Tentative * Cleanup the cleanup * Rm reference to the micronaut server * format * pmd * more pmd * fix build * Delete logs API * Revert "Delete logs API" This reverts commit fcb271dcdf51d9904fa5d482e5c3e6984d5bc182. * Rm flaky test * Format * Try to fix test * Format * Remove optional * Rm import * Test sleep * Simplify injection * update import * Remove sleep * More injection * Remove more requirement * imports * Remove more requirement * Fix yaml * Remove unused conf * Add role * Test acceptance test * Update env * Revert "Update to Micronaut 3.8.0 (#20716)" This reverts commit a28f93747ffb7c8ad9e30a4a90ceada9d0d7dc0e. * Update helm chart * Fix helm chart * Convert Application Listener * Format * Add explicit deployment mode * Change check port * Update version and bump version to the right value * Cleanup * Update FE end to end test * Allow head request * Fix controller * Format * Fix http client Bean * Format Co-authored-by: Cole Snodgrass * 🪟 🐛 🎨 Fix stream table icon checkboxes and icons (#21108) * Update new stream table icon positioning and style * Add custom icons from design file * Add opacity animation * Fix checkbox sizing in current streams table * Update test snapshots * Use explicit configuration for Micronaut endpoints (#20759) * process config control messages during `check` and `discover` (#20894) * track latest config message * pass new config as part of outputs * persist new config * persist config as the messages come through, dont set output * clean up old implementation * accept control messages for destinations * get api client from micronaut * mask instance-wide oauth params when updating configs * defaultreplicationworker tests * formatting * tests for source/destination handlers * rm todo * refactor test a bit to fix pmd * fix pmd * fix test * add PersistConfigHelperTest * update message tracker comment * fix pmd * format * move ApiClientBeanFactory to commons-worker, use in container-orchestrator * pull out config updating to separate methods * add jitter * rename PersistConfigHelper -> UpdateConnectorConfigHelper, docs * fix exception type * fmt * move message type check into runnable * formatting * pass api client env vars to container orchestrator * pass micronaut envs to container orchestrator * print stacktrace for debugging * different api host for container orchestrator * fix default env var * format * fix errors after merge * set source and destination actor id as part of the sync input * fix: get destination definition * fix null ptr * remove "actor" from naming * fix missing change from rename * revert ContainerOrchestratorConfigBeanFactory changes * inject sourceapi/destinationapi directly rather than airbyteapiclient * UpdateConnectorConfigHelper -> ConnectorConfigUpdater * rm log * fix test * dont fail on config update error * process control messages for discover jobs * process control messages for CHECK * persist config updates on check_connection_for_update * get last config message rather than first * fix pmd * fix failing tests * add tests * source id not required for check connection (create case) * suppress pmd warning for BusyWait literal * source id not required for checkc onnection (create case) (p2) * pass id, not full config to runnables/accept control message * add new config required for api client * add test file * remove debugging logs * rename method (getLast -> getMostRecent) * rm version check (re-added this in by mistake on merge) * fix test compatibility * simplify * 🪟🧹 Connector builder: Fast fields performance improvements (#20957) * improve some types * improve further * clean up a bit more * refactor loading state * move loading state up * remove isLoading references * remove unused props and make fetch connector error work * remove special component for name * remove top level state for unifinished flows * start removing uiwidget * Update airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss Co-authored-by: Tim Roes * remove undefined option for selected id * remove unused prop * fix types * remove uiwidget state * clean up * adjust comment * handle errors in a nice way * do not respect default on oneOf fields * rename to formblock * reduce re-renders * pass error to secure inputs * simplify and improve styling * align top * code review * remove comment * review comments * rename file * be strict about boolean values * add example * track form error in error boundary * review comments * handle unexpected cases better * speed up some bits * more changes * enrich error with connector id * 🪟🎉 Add copy stream button (#20577) * add copy stream button * review comments * rename prop * 🪟🎉 Connector builder: Integrate connector form for test input (#20385) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * fix small stuff * add warning label * review comments * adjust translation Co-authored-by: lmossman * use request_body_json instead of request_body_data * :window: :art: Move `Add` button into the line of Connector Builder key value list fields (#20699) * move add button into line * add stories for empty with control, and content + control * change button name to Control * 🪟🎉 Connector builder: Allow defining inputs (#20431) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * handle stored form values that don't contain new fields properly * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * 🪟🎉 Connector builder authentication (#20645) * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * fix keys * 🪟🎉 Connector builder: Session token and oauth authentication (#20712) * session token and oauth authentication * fill in session token variable * typos * make sure validation error does not go away * 🪟🎉 Connector builder: Always validate inputs form (#20664) * validate user input outside of form * review comments Co-authored-by: lmossman Co-authored-by: lmossman * fix merge conflict with dropdown prop being renamed to control * [Connector Builder] Add paginator (#20698) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly Co-authored-by: Joe Reuter * [Connector Builder] Add stream slicer (#20748) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly * save stream slicer progress * finish stream slicer * fix stream slicer fields and validation Co-authored-by: Joe Reuter * debounce form builder values update to reduce load * 🪟🔧 Connector builder: use new lowcode manifest (#20715) * use new manifest yaml * Update airbyte-webapp/src/components/connectorBuilder/types.ts Co-authored-by: Lake Mossman * use updated manifest types Co-authored-by: Lake Mossman * debounce validation as well * akways show stream test button in error state if there are errors * fix type of oauth input * review comments * fix more * start implementing fast fields * a few more fastfields * fix some stuff * complete fast fields work * memoize a bit more * remove irrelevant change * fix types * some more improvements * prevent all out-of-line validation calls * use derivedJsonManifest when in UI view Co-authored-by: Tim Roes Co-authored-by: lmossman * In Cloud, out of date connectors call always returns 0 (#21126) * isCloudApp returns a boolean now * Connector service never makes the check updates call in Cloud * Move hadoop-lzo to test dependency (#21085) * remove hadoop-lzo from gradle dependencies in destination-s3 project * move hadoop-lzo from build time to test time dependencies in base-s3 project * Update Connector Builder docs to reflect configuration UI additions (#21129) * update connector builder docs for UI MVP release * remove note about separately declaring schemas * Revert "Convert the server to micronaut (#19194)" (#21132) This reverts commit 0b153d11f9b5cc723acc6bbfd0478bd74e124c53. * 🐛 Source Google Ads: publish missing `0.2.6` (#21103) * Remerge Progress Bar Read API. (#21124) Let's try #20937 again, this time with better test for error cases. See original PR for description. This PR adds testing and logic to handle empty/bad job input. * Bump Airbyte version from 0.40.26 to 0.40.27 (#21135) Co-authored-by: lmossman * add import rule changes for Kotlin code (#21136) * Source Okta: OAuth2.0 support - disabled (#20877) * Disabled OAuth Okta * Updated acceptance tests * Updated PR number * auto-bump connector version Co-authored-by: Octavia Squidington III * Remove the quarantine status (#21088) * Rm temporal version * Remove temporal version * Update the replayed workflow * Remove quarantine information * normalization: delete supportsDbt and supportsNormalization from DestinationDefinitionSpecificationRead (#21005) * delete supportsDbt and supportsNormalization from DestinationDefinitionSpecificationRead * FE changes * add new fields to the DestinationDefinitionRead API model * update handlers * update api doc * update handlers * remove debug loggin * implement suggestions * update octavia-cli tests * fix supported check * update octavia-cli * fix integration tests * fix mocks * fix forgotten renaming Co-authored-by: Evan Tahler * Allow for custom requesters to be defined in low-code manifests (#21001) * Allow for custom requesters to be defined in low-code manifests * add test for custom requester component * bump versions and changelog * adjust readme (#20945) * 🪟🎨 Connector form: Improve logs look and feel (#20951) * improve logs look and feel in connector form * add button to toggle * lokk and feel * Adjust GitHub flow for FE chapter (#20813) * Adjust GitHub flow for FE chapter * Change labeler config * Source: Google Analytics 4 (GA4) - improve config validation and SAT (#20889) Signed-off-by: Sergey Chvalyuk * Simplify messaging for sync canceled jobs (#20999) * Hide stats * Show cancelled icon when it's multiple attempts * Update attempts count to gray * Extract shared attempt functions into utils file * Cleanup component export and scss imports * Fix height glitch when opening and closing log * Remove deprecated getter (#21089) * Rm temporal version * Remove temporal version * Update the replayed workflow * Remove quarantine information * Remove deprecated getter * Import error broke master (#21165) * allow the cursors and primary keys to be deselected when the sync mode does not require it (#21117) * Connector builder server: Add inferred schema to read API response (#20942) * fix stuff * add inferred schema to API * fix yaml changes * fix yaml formatting * add whitespace back * reorder imports * New destination: databend (community PR #19815) (#20909) * feat: Add databend destination Co-authored-by: hantmac Co-authored-by: josephkmh Co-authored-by: Sajarin * Update postgres.md (#21170) * Source Facebook Marketing: Update schema ad_account (#21149) * Source Facebook Marketing: Update schema ad_account * Source Facebook Marketing: Update docs; bump version * Source Facebook Marketing: disable backward compatibility * auto-bump connector version Co-authored-by: Octavia Squidington III * Revert "Revert Convert the server to micronaut" (#21133) * Revert "Revert "Convert the server to micronaut (#19194)" (#21132)" This reverts commit 31c65f8eea479ba7e1149f2ccb936220b27a39f1. * Fix the cors * Fix cloud * Amazon Seller Partner: Validate connections without orders data (#20896) * amazon seller partner validate connection without orders data * amazon seller partner bump version in docs * amazon seller partner fix redundant if statement * auto-bump connector version Co-authored-by: Sajarin Co-authored-by: Octavia Squidington III * Source Harvest: Skip 403 FORBIDDEN for all streams (#21151) * Source Harvest: Skip 403 FORBIDDEN for all streams * Source Harvest: update docs * auto-bump connector version Co-authored-by: Octavia Squidington III * test for behavior when a column is removed in an incremental sync (#20033) * test for behavior when a column is removed in an incremental sync * fixes in dat test for dropping a column * only run drop-one-column test for the exchange rates dataset * re-enable tests that were disabled during development * remove unused import * update test to new method for checking spec capabilities * use config directly instead of parameterized test Co-authored-by: Michael Siega Co-authored-by: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> * Source Mailchimp - fix the way request params are built (#20765) * 1058 source mailchimp - fix the way request params are built * auto-bump connector version Co-authored-by: Octavia Squidington III * 🐛 🎉 Source Airtable: migrate to the `Metadata API` for dynamic schema generation (#20846) * Extend connection list filtering (#21094) * Update listWorkspaceStandardSyncs to support lists of sources/destinations * Update API spec * test cleanup (#21178) * removed error (#20612) * Adjust connector icons (#20547) * Adjust connector icons * Update icons * Add missing icons * Remove missing icon from spec * Add trailing newline * Fix test * Use jest directly as test runner (#21174) * 🪟 🎨 Refining BulkEditPanel component (#20474) * Adds handling for source defined and unavailable stream paths inside PillSelect * Adds handling of unavailable cursor and primary key field for PillButton components * Adds useMemo to SyncPathSelect component * Add InfoText component for cursor field and primary key to match table design in Figma * Removes animation for BulkEditPanel; Adds StreamPathSelect new property pillIfChangeUnavailable for cases when we need or not to render pill under unavailable to change path text * Disables pointer event while PillButton is disabled * Adds renderDisabledState property to control PillSelect content while disabled * Changes renderDisabledState -> disabledLabel * 🪟 🔧 Add tests for BulkEditService (#20820) * Add tests for BulkEditService * Add act function to test cases while updating state * Add tests full description; Adds type to MOCK_NODES array; Fixes import relative path * Connector builder: E2e tests (#21122) * wip * wip * e2e tests for connector builder server * rename function * clean up * clean up a bit more * fix path * fix and add documentation * more documentation * stabilze * review comments * 🎉 Source Airtable: update `releaseStage` to `beta` (#20935) * 🪟 🎉 Disable deselection of cursor field/primary key in the UI (#20844) * 🪟 🔧 Update frontend typing of cloud api response (#21086) * 🪟 🔧 Use CSS Custom Properties for colors (#19344) * use css custom properties for colors * separate tokens by hue * adjust stylelint * fix code editor color transformations * remove blue-transparent from color palette * scope opacity to bg only * rename to theme-light * avoid transparent blue, use palette color instead * bring back expanding hex values * replace tooltip colors * define overlay color in theme * fix tooltip colors * slightly darker link color * remove unused SC colors * add box shadow variables * light tooltip table variant * inline one-off colors for login * add new colors to theme-light * fix import name * fix merge conflict with box shadows * fix overlay bg color Co-authored-by: lmossman * ci-connector-ops: change required reviewers logic (#21158) * [ISSUE-20322] updating tutorial documentation (#21163) * [ISSUE-20322] updating tutorial documentation * [ISSUE-20322] updating test and improving docs * Connector builder server: Fix unit tests (#21107) * fix stuff * add inferred schema to API * fix yaml changes * fix yaml formatting * add whitespace back * fix tests * reformat * add unit tests * Destination connector for teradata vantage (rebased community PR) (#21160) * Destination connector for teradata vantage is implemented * add teradata destination to seed yaml * Add teradata destination to specs seed file Co-authored-by: Satish Chinthanippu * 🎉 Updates for edit password field in connector (#20723) * Moves Cancel button to the left. * changes Edit button to outline - secondary variant. * Updates button size to match input height. * convert EnvVariableFeatureFlag to an injected dependency (#21171) * inject feature-flag client into integration-launcher * inject feature-flag client into message-tracker * inject feature-flag client into DefaultAirbyteSource * missed reference in message-tracker * Updated ISO certification date (#21181) * added info * edited wording * Source google ads: add more logs (#20755) * #1148 source google ads: add more logs * #1148 source google ads: upd changelog * #1148 source google ads: flake fix * #1148 source google ads - fix SATs * #1148 source Google Ads: bump version * #1148 source google ads: upd expected records * auto-bump connector version Co-authored-by: Octavia Squidington III * 🎉 Destination Local CSV: add custom delimiter (#17998) * Adds delimeter to the spec file. Adds function to get delimeter from the config. * New delimiter works, but only checked raw airbyte tables. * Fixes testWriteFailure(), testWriteSuccess() still broken. * Corrects CSVFormat and now all tests pass. * Implements tab separator. * Corrects tooltip on destination settings page. * Creates CSV file with delimiters and prints it as stirng. * Adds try catch block for assertion. Deletes file after test run. * Removes separate format for tab dleimiter, it is not needed. * Cleans up code. * Adds missing bracket. * Adds files from incorrect rebase. * Corrects imports. * Fixes connectors base build. * Corrects Dockerfile version bump. Adds changelog. * Corrects getProtocolVersion method and makes CSVDataArgumentsProvider class static. * auto-bump connector version Co-authored-by: Octavia Squidington III * Source Airtable: fix field names (#21215) * Source S3: update block size for json (#21210) * Source S3: update block size for json * Source S3: update docs * auto-bump connector version Co-authored-by: Octavia Squidington III * 🪟 🧹 Custom connectors in Cloud UI updates (#21034) * check if doc url is a valid url in cloud, but don't require it * field label and placeholder * cleanup * match casing for other form labels * cleanup imports * hide documentation panel if URL is empty * only show documentation panel if url is from our docs * make docs url optional, don't put a link in conncectorsview if empty * Update airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx Co-authored-by: Joey Marshment-Howell * use , cleanup from Joey's commit in review comment * Use component in ImageCell, get styled components out of there while I'm at it * add "required" labels * show validation error regardless of `touched` for documentationUrl * use old placeholder instead Co-authored-by: Joey Marshment-Howell * lowcode docs: add link to YAML reference (#21231) * Update platform-workflow-labels.yml (#21225) * Update README.md (#21236) Update connector release stage docs * Enforce HTTPS (#21182) * Enforce HTTPS * 🪟🐛 Connector builder UI: Fix datetime stream slicer (#21161) * fix datetime stream slicer * Update airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx Co-authored-by: Lake Mossman Co-authored-by: Lake Mossman * [ISSUE #19981] testing version bump (#21106) * [ISSUE #19981] testing version bump * [ISSUE #19981] debugging part-to-bump * [ISSUE #19981] further debugging on part-to-bump * [ISSUE #19981] further debugging on part-to-bump yet again * [ISSUE #19981] further debugging on part-to-bump yet again and again * [ISSUE #19981] improving on first successful attempt * [ISSUE #19981] adding changelog and concurrency * [ISSUE #19981] update version in .bumpversion.cfg * [ISSUE #19981] testing Slack notification with working channel * [ISSUE #19981] documentation and clean up before PR * [ISSUE #19981] make changelog-message optional * [ISSUE #19981] align version with new release * [ISSUE #19981] code review * [Low-Code CDK] Handle forward references in manifest (#20893) [Low-Code CDK] Handle forward references in manifest * Update form field labels in connection form to match design (#21036) * Update form field labels in connection form to match design - Replaced ControlLabels message property with infoTooltipContent - Replaced property "flex-start" to "center" for FlexContainer (New Connection) * Hide no credits banner for no billing accounts (#21218) * Fix auto detect schema change backdrop color CSS (#21246) * Source notion: fix schema (#20639) * #1047 oncall - Source Notion: fix Pages stream schema * #1047 oncall - Source Notion: fix Pages stream schema * #1047 oncall - Source Notion: upd changelog * #1047 source notion - remove ignored fields * #1047 source notion: fix SATs * #1047 source notion: upd expected records * auto-bump connector version Co-authored-by: Octavia Squidington III * [Connector-builder server] Allow client to specify record limit and enforce max of 1000 (#20575) * Upgrade to Micronaut 3.8.0 and related dependencies (#21077) * Upgrade to Micronaut 3.8.0 and related dependencies * Define hostname/subdomain to be compatible with updated AWS SDK * Use bucket name for hostname * Pin dependencies to avoid behavior change * Add comment * Source Google Analytics Data API: slicer updated, unit tests added (#21169) Signed-off-by: Sergey Chvalyuk * Bmoric/restore old interface (#21235) * Re introduce old exception mapper * Use mapper * Switch CI to use npm ci (#21259) * postgres-source-cdc: handle null values for array data types (#21003) * postgres-source-cdc:handle null values for array data types * fix test * upgrade version * auto-bump connector version Co-authored-by: Octavia Squidington III * 🪟 🐛 Fix broken switch UI state - when checked state provided as a "value" (#21219) * fix broken switch state - when checked state provided as a "value" * Update airbyte-webapp/src/components/ui/Switch/Switch.tsx Co-authored-by: Krishna (kc) Glick Co-authored-by: Krishna (kc) Glick * 🪟 🔧 Use experiment for new streams table design (#21230) * Use experiment for new streams table design * Update formConfig validationSchema to use a hook to get the feature flags directly * Add useNewTableDesignExperiment helper hook * Rename isNewStreamsTableEnabled -> isNewTableDesignEnabld * Cleanup tidyFormValues to take the existing schema in * Fix var name in CatalogTreeTableHeader * Source Snapchat Marketing: fix error response (#21267) * Source Snapchat Marketing: fix error response * Source Snapchat Marketing: update docs * auto-bump connector version Co-authored-by: Octavia Squidington III * [Low-Code CDK] Parse incoming manifest objects into Pydantic models (#20747) * initial work to parse manifest objects into pydantic models * pr feedback and some other validations * rerun manifest schema generation * remove field constraint due to bug * add a few tests for the model to component factory * add catch * fix a bug where propagated schema doesn't work with old factory * add additionaProperties: false to the handwritten schema * regenerate pydantic models * fix test * [Low Code CDK] Add type annotations and cleanup usage of `visited` set (#21268) * [Low Code CDK] Add type annotations and cleanup usage of `visited` set * 🤖 Bump minor version of Airbyte CDK * feat: mention user on failed master branch build (#21201) * 🪟 🔧 Fixed passed props to `` which cause error in console (#20711) * fixed sourceDefinitionId and destinationDefinitionId * clean up props * Connector builder server: Fail on failing tests (#21198) * fail on failing command * fix test * Source Google Ads: Update docs (#21308) * Source Ads: docs * Source Ads: docs * Migrate airbyte-bootloader to Micronaut (#21073) * Migrate airbyte-bootloader to Micronaut * PR feedback * More PR feedback * Rename variable for clarity * Add properties to support cloud * Formatting * Use default values for env vars * Re-organization to support cloud overrides * Disable conditional logging * More singleton cleanup * test: try CI without fluentbit * Revert "test: try CI without fluentbit" This reverts commit 8fa0f7410602078296c859c0e783dbdc8456c4a7. * test: enable SSH on EC2 runner * Revert "test: enable SSH on EC2 runner" This reverts commit e4867aae0965faa6ebf83e19e5bd9e9765478099. * Avoid early database connection on startup * Fix compile issues from refactor * Formatting Co-authored-by: perangel * 🪟 🧹 Reorganize connections pages (#20845) * Reorganize connection pages Flatten structure Simplify names * Consolidate all the Connection routes to a single file * Extract job types and utilities to own files within JobItem Move JobsWithJobs to JobItem/utils * Move ConnectionName component to components/connection * Move StatusMainInfo components and rename to components/connection/ConnectionInfoCard * Move ConnectionBlock to components/connection * Clean up ConnectionPage structure * Update style import in ConnectionInfoCard test * Clean up ConnectionRoutePaths enum * Apply suggestions from code review - Update export default style Co-authored-by: Krishna (kc) Glick * Update ConnectionInfoCard test description name * Update test snapshots Co-authored-by: Krishna (kc) Glick * Add bootloader/server to published images for Cloud (#21325) * 🪟 🚦 E2E: Fix duplicated database object warning (#20926) * replace "cypress-postgres" with "pg-promise" npm package * update cypress config * update userDefineConnection type and naming * Update airbyte-webapp-e2e-tests/cypress/plugins/index.ts Co-authored-by: Mark Berger Co-authored-by: Mark Berger * Run Airbyte CI on pull_request (#21266) * Run Airbyte CI on pull_request as well * also only run on pushes to master * 🎉 Source bigcommerce: add brands and categories streams (#20518) * Add product info to the orders stream * Update docs * Refactor order_products into its own stream * formatting * Update docs and docker version * remove duplicate field * Add brands and categories streams to bigcommerce * remove duplicate state * remove duplicate source * update docs * update docs * bump docker version * update markdown * lint * auto-bump connector version Co-authored-by: Sajarin Co-authored-by: Haithem SOUALA Co-authored-by: Octavia Squidington III * 🎉 New Destination: Weaviate (#20094) * Add Weaviate Destination #20012 * Fix formatting and standards * Fix flake issue * Fix unused client variable * Add support for int based ID fields * Ensure stream name meets Weaviate class reqs * add integration test for using pokemon as source * handle nested objects by converting to json string * create schema for transforming data to weaviate * Add docs for weaviate destination * Remove pokemon-schema external dependency * Remove pikachu integration test external dep * Add large batch test case * add test for second sync * Fix issue with fields starting with uppercase * add more checks to line_break test * Update README for Weaviate * Make batch_size configurable with 100 as default * Add support for providing vectors * Update docs * Add test for existing Weaviate class * Add trying to create schema in check connection * Add support for mongodb _id fields * Add support for providing custom ID * remove unused file * fix flow of is_ready() check * Move standalone functions to utils.py * Support overwrite mode * Add regex based stream_name_class_name conversion * remove unneeded print statement * Add "airbyte_secret" : true to password config * add support for array of arrays * remove unneeded variable declaration * change to MutableMapping since we use del * change name from queued_write to buffered_write * add retry on partial batch error * Fix partial batch retry and add tests * fix ID generation * Clean up recursive retry logic * fix flake tests * ran flake reformat * add definitions Co-authored-by: Ivica Taseski Co-authored-by: itaseski * Make schema field in source-snowflake mean a subset of the specified o… (#20465) * Make schema field in source-postgres mean a subset of the specified of schema when during discover(). update UI * Add missing file * Fix failing acceptance test * sanity * update doc * typo * version bump and release note * Fix failing test * fix format * Ensure Local Secret Persistence is Initialized (#21329) * Formatting * Ensure that local persistence is initialized * December release notes docs (#21270) * added december release notes * edits * Add dependencies to docker-compose (#19321) * Revert "Revert "Add dependencies to docker-compose (#19257)" (#19306)" This reverts commit b16f28fd4d9bd05c3512d1f6dbb94af2672259b3. * Bump docker-compose version * test * Revert "test" This reverts commit c3b28c9ec3233d33c2646b349f02a09fbd520f65. * Bump docker compose version to 3.8 * test * test * Use a newer docker for platform build * test * Use docker compose v2 * Update docker for frontend e2e * Use docker compose v2 for e2e tests * Update tools to use docker compose v2 * Update more docker compose references to v2 * Update docker compose usage in docs * Update deploying airbyte docs * Fix permission * Case insensistive check * Add extra mention for docker compose in upgrading airbyte * Improve upgrade docker note * store env-var response (#21327) * store env-var response * fix formatting * fix pmd issue * change log message to debug * fix file change error * Better escape the select query for a case a schema name starts with a number (#21051) * Better escape the select query for a case a schema name starts with a numebr. * improve test * fix unrelated build error * manually bump source definition version (#21349) * :bug: Source Jira: `check_connection` fails if no projects are defined (#20739) Co-authored-by: Evan Tahler Co-authored-by: Octavia Squidington III Co-authored-by: Serhii Chvaliuk Co-authored-by: Augustin * airbyte-common-workers: Collect trace message on failed connection_status (#20721) * 🪟 🎉 Verify auth status on tab focus (#21175) * Make documentation_url optional in a declarative connector spec (#21347) * Make documentation_url optional in a declarative connector spec * simplify if statement * bump patch version of cdk * Revert "bump patch version of cdk" This reverts commit 1854bf3be1ee71500b95053ebd9b595a71b3d693. * 🤖 Bump patch version of Airbyte CDK * Revert "🤖 Bump patch version of Airbyte CDK" This reverts commit 85d5a989e2a8dde04342ee31e8dbc03df5887154. * 🤖 Bump patch version of Airbyte CDK Co-authored-by: lmossman * Avoid parsing a message if this message is too big (#21090) * Tmp * Format * TMP * TMP * Inject max memory * Clean up * Improve error message * PR comments * Unrelated changes * Fix pmd * JDBC Destinations: improve error message for conflicting streams (#21342) * catch conflicting streams as configerror * add test * bump version + changelog * derp, fix test setup * derp * auto-bump connector version Co-authored-by: Octavia Squidington III * Bump Airbyte version from 0.40.27 to 0.40.28 (#21359) Co-authored-by: lmossman * Add records committed to job stats (#21364) * [Low-Code CDK] Construct declarative components from Pydantic models (#21050) * initial work to parse manifest objects into pydantic models * pr feedback and some other validations * rerun manifest schema generation * remove field constraint due to bug * initial work to construct most components from greenhouse * custom components parse subcomponent fields correctly and adding a few more component constructors * construct components from gnews * first pass at posthog.yaml * Handle nested custom components with list values. Also includes updates to posthog.yaml, including autoformatting changes. * adding constructors for slicers, filters, and transformations and a few bug fixes * make sed work across multiple OS * add NoAuth component * fix handling of custom components with nested list * Autogenerate `TYPE_NAME_TO_MODEL` mapping * Handle default kwargs not defined on model for custom components * Re-add `options` for CartesianProductStreamSlicer for backwards compat with custom stream slicers * add basic unit tests for the model component factory * add back defaults and extra parameters like options to retain compatibility with legacy flow and backwards compatibility * Remove `_get_defaults`; using actual default values on classes instead * Add backoff strategy component creation functions * add back defaults and extra parameters like options to retain compatibility with legacy flow and backwards compatibility * add lots of tests to construct components from the pydantic models and a few bug fixes * add a few tests for the model to component factory * add catch * fix a bug where propagated schema doesn't work with old factory * clean up a few files * add type inference for custom components, more tests and some refactoring of the model factory * template, docs, manifest updates, pr feedback and some cleanup * pr feedback and polish schema a bit * fix tests from the latest rebase of master * fix the last few bugs I found and adjust a few sources that weren't perfectly compatible with the new component flow * fix CheckStream bug cleanup and a few small tweaks and polish * add additional test to cover bug case * fix formatting * 🤖 Bump minor version of Airbyte CDK Co-authored-by: Catherine Noll Co-authored-by: brianjlai * update destination-python generator (#21298) * fixed current_value in usage_triggers stream (#21304) * 🪟 🔧 Add stylelint plugin & no-color-variables-in-rgba rule (#21269) * fix e2e tests (#21380) * 🪟🎉 Connector builder: Schema inferrer UI (#21154) * fix stuff * add inferred schema to API * fix yaml changes * fix yaml formatting * add whitespace back * basic ui * advanced UI * Remove unused one * reset package lock * resolve merge conflicts * styling * show button and icon in the normal schema tab * restructure * handle yaml view * small fix * review comments * make monaco resize * review comments * 🎉 New Source: Dremio (#19912) * Remove `supportsNormalization` `supportsDbt` from non GA destination spec (#21319) * SAT: check `future_state` only for incremental streams. (#21248) Signed-off-by: Sergey Chvalyuk * :tada: Typeform - beta certification (#21145) * fixed SAT, fix specs, updated docs * fix incremental * fix incremental * fix incremental * updated connector version in source_definitions.yaml * updated version in source_specs.yaml * Update databend.md (#21179) * Update databend.md * component (#21273) * genericize alert component * add className prop to alert * rename infobox "callout", swap in place for scss module * fix imports * use callout in place of alert * add blue variant, remove icon prop * remove font-size (this is already the default) * remove single use prop, use a classname instead * use semantic names, remove empty line * cleanup from rebase * Update to Micronaut 3.8.1 (#21388) * Source Slack: retry timeout error (#21321) * Source Slack: retry timeout error * Source Slack: update docs * Source Slack: update retry * auto-bump connector version Co-authored-by: Octavia Squidington III * Source Facebook Marketing: Videos stream remove filtering (#21385) * Source Facebook Marketing: Videos stream remove filtering * Source Facebook Marketing: update docs * Source Facebook Marketing: Ref * auto-bump connector version Co-authored-by: Octavia Squidington III * Source Klaviyo: enable SAT high strictness (#21374) * Source Klaviyo: change configuration of acceptance-test-config.yml for high strictness * Add metrics stream to configured_catalog, add metrics expected records * Update future state * Remove expected records for flows stream * Source File: enable high strictness level (#21243) * Source File: enable high strictness level * Addedend line Co-authored-by: Serhii Lazebnyi * introduce code coverage report step (#21376) * introduce code coverage report step * remove airbyte-queue jacoco report since it isn't being generated by the build yet * leave pointer to wildcard support and tweak coverage threshold * added edits (#21346) * 🪟 🐛 Fix jest tests to fail build correctly (#21408) * Fix jest tests to fail build correctly * Add failing test for demo * Update StatusCell.test.tsx * Allow the OSS server only on OSS (#21416) * Update create connection page to full width, and fix trivial UI issues (#21352) * Update big prop in FormPageContent to expand to full width * Fix Normalization & Tranformation heading size * Update FormPageContent to scss and add bottom padding * Remove max-width from ConnectionReplicationPage * Improvements to edge cases of CheckStream (#21404) * Add test for failure case * Except StopIteration - make test pass * Don't attempt to connect to a stream if we get no stream slices * Make helper method for getting first record for a slice * Add comments and exit early if stream to check isn't in list of source streams * move helpers to helper module * Clarify what it means when StopIteration is returned by helper methods * Add AllowedHosts to Actor Definitions and Config Database (#21363) * Add AllowedHosts to actor_definitions and database * use objects for better null-ness handling * Tables.ACTOR_DEFINITIO * Update on-gcp-compute-engine.md (#21429) Use curl instead of wget since we already install curl. * refactor checks * rebase from master * PR Feedback * PR feedback * revert unintended version downgrade * add version updates and changelog * auto-bump connector version * auto-bump connector version * formatting * bump snowflake destination version * manual version bump for s3 * auto-bump connector version * auto-bump connector version Signed-off-by: Sergey Chvalyuk Co-authored-by: Joe Reuter Co-authored-by: Tim Roes Co-authored-by: lmossman Co-authored-by: Anne <102554163+alovew@users.noreply.github.com> Co-authored-by: Greg Solovyev Co-authored-by: Octavia Squidington III Co-authored-by: Davin Chia Co-authored-by: Benoit Moriceau Co-authored-by: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Co-authored-by: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Co-authored-by: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Co-authored-by: Serhii Chvaliuk Co-authored-by: Augustin Co-authored-by: Topher Lubaway Co-authored-by: Krishna (kc) Glick Co-authored-by: Pedro S. Lopez Co-authored-by: Maxime Carbonneau-Leclerc Co-authored-by: Cole Snodgrass Co-authored-by: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Co-authored-by: Jonathan Pearlin Co-authored-by: Baz Co-authored-by: Octavia Squidington III <90398440+octavia-squidington-iii@users.noreply.github.com> Co-authored-by: lmossman Co-authored-by: Evan Tahler Co-authored-by: Brian Lai <51336873+brianjlai@users.noreply.github.com> Co-authored-by: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Co-authored-by: hantmac Co-authored-by: josephkmh Co-authored-by: Sajarin Co-authored-by: Zaza Javakhishvili Co-authored-by: Michael Siega Co-authored-by: Denys Davydov Co-authored-by: Jimmy Ma Co-authored-by: Sophia Wiley <106352739+sophia-wiley@users.noreply.github.com> Co-authored-by: Yatsuk Bogdan Co-authored-by: Joey Marshment-Howell Co-authored-by: Satish Chinthanippu Co-authored-by: Nataly Merezhuk <65251165+natalyjazzviolin@users.noreply.github.com> Co-authored-by: Teal Larson Co-authored-by: Sherif A. Nada Co-authored-by: Catherine Noll Co-authored-by: Mark Berger Co-authored-by: Subodh Kant Chaturvedi Co-authored-by: Vladimir Co-authored-by: Volodymyr Pochtar Co-authored-by: perangel Co-authored-by: Conor Co-authored-by: Philip Corr Co-authored-by: Haithem SOUALA Co-authored-by: Sam Stoelinga Co-authored-by: Ivica Taseski Co-authored-by: itaseski Co-authored-by: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Co-authored-by: Philippe Boyd Co-authored-by: Edward Gao Co-authored-by: Catherine Noll Co-authored-by: brianjlai Co-authored-by: darynaishchenko <80129833+darynaishchenko@users.noreply.github.com> Co-authored-by: Arsen Losenko <20901439+arsenlosenko@users.noreply.github.com> Co-authored-by: midavadim Co-authored-by: Roman Yermilov [GL] <86300758+roman-yermilov-gl@users.noreply.github.com> Co-authored-by: Serhii Lazebnyi Co-authored-by: Ella Rohm-Ensing --- .../seed/destination_definitions.yaml | 10 ++-- .../resources/seed/destination_specs.yaml | 10 ++-- .../destination/s3/BlobStorageOperations.java | 4 +- .../destination/s3/S3BaseChecks.java | 2 +- .../destination/s3/S3StorageOperations.java | 26 +++++---- .../util/ConnectorExceptionUtil.java | 3 + .../Dockerfile | 2 +- .../destination-bigquery/Dockerfile | 2 +- .../bigquery/BigQueryGcsOperations.java | 15 ++++- .../bigquery/BigQueryDestinationTest.java | 4 +- .../connectors/destination-gcs/Dockerfile | 2 +- .../destination-redshift/Dockerfile | 2 +- .../RedshiftS3StagingSqlOperations.java | 2 +- .../connectors/destination-s3/Dockerfile | 2 +- .../destination-snowflake/Dockerfile | 2 +- .../SnowflakeS3StagingSqlOperations.java | 2 +- docs/integrations/destinations/bigquery.md | 1 + docs/integrations/destinations/gcs.md | 57 ++++++++++--------- docs/integrations/destinations/redshift.md | 3 +- docs/integrations/destinations/s3.md | 1 + docs/integrations/destinations/snowflake.md | 3 +- 21 files changed, 90 insertions(+), 65 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index d84582706ba2..b05591295659 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -40,7 +40,7 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 1.2.11 + dockerImageTag: 1.2.12 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg normalizationConfig: @@ -139,7 +139,7 @@ - name: Google Cloud Storage (GCS) destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a dockerRepository: airbyte/destination-gcs - dockerImageTag: 0.2.12 + dockerImageTag: 0.2.13 documentationUrl: https://docs.airbyte.com/integrations/destinations/gcs icon: googlecloudstorage.svg resourceRequirements: @@ -290,7 +290,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.53 + dockerImageTag: 0.3.54 documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift icon: redshift.svg normalizationConfig: @@ -321,7 +321,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.3.18 + dockerImageTag: 0.3.19 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3 icon: s3.svg resourceRequirements: @@ -348,7 +348,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.43 + dockerImageTag: 0.4.44 documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake icon: snowflake.svg normalizationConfig: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index aa768b304061..7f34e408d5b9 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -621,7 +621,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-bigquery:1.2.11" +- dockerImage: "airbyte/destination-bigquery:1.2.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery" connectionSpecification: @@ -2325,7 +2325,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-gcs:0.2.12" +- dockerImage: "airbyte/destination-gcs:0.2.13" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/gcs" connectionSpecification: @@ -5123,7 +5123,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.53" +- dockerImage: "airbyte/destination-redshift:0.3.54" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift" connectionSpecification: @@ -5492,7 +5492,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.3.18" +- dockerImage: "airbyte/destination-s3:0.3.19" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3" connectionSpecification: @@ -6109,7 +6109,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.4.43" +- dockerImage: "airbyte/destination-snowflake:0.4.44" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/snowflake" connectionSpecification: diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java index d131470e976e..96681b4216f8 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java @@ -22,9 +22,9 @@ protected BlobStorageOperations() { public abstract String getBucketObjectPath(String namespace, String streamName, DateTime writeDatetime, String customFormat); /** - * Create a storage object where to store data in the destination for a @param objectPath + * Ensure that the bucket specified in the config exists */ - public abstract void createBucketObjectIfNotExists(String objectPath) throws Exception; + public abstract void createBucketIfNotExists() throws Exception; /** * Upload the data files into the storage area. diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java index b5f2037e842b..0873cbb87a01 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java @@ -120,7 +120,7 @@ private static void attemptWriteAndDeleteS3Object(final S3StorageOperations stor final var bucketPath = s3Config.getBucketPath(); if (!Strings.isNullOrEmpty(bucketPath)) { - storageOperations.createBucketObjectIfNotExists(bucketPath); + storageOperations.createBucketIfNotExists(); } s3.putObject(s3Bucket, outputTableName, "check-content"); testIAMUserHasListObjectPermission(s3, s3Bucket); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java index 599463435142..ba9f3a19ceab 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java @@ -8,6 +8,7 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.ListObjectsRequest; @@ -15,12 +16,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.string.Strings; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateManager; import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.integrations.util.ConnectorExceptionUtil; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -95,23 +98,15 @@ public String getBucketObjectPath(final String namespace, final String streamNam /** * Create a directory object at the specified location. Creates the bucket if necessary. - * - * @param objectPath The directory to create. Must be a nonempty string. */ @Override - public void createBucketObjectIfNotExists(final String objectPath) { + public void createBucketIfNotExists() { final String bucket = s3Config.getBucketName(); - final String folderPath = objectPath.endsWith("/") ? objectPath : objectPath + "/"; if (!doesBucketExist(bucket)) { LOGGER.info("Bucket {} does not exist; creating...", bucket); s3Client.createBucket(bucket); LOGGER.info("Bucket {} has been created.", bucket); } - if (!s3Client.doesObjectExist(bucket, folderPath)) { - LOGGER.info("Storage Object {}/{} does not exist in bucket; creating...", bucket, objectPath); - s3Client.putObject(bucket, folderPath, ""); - LOGGER.info("Storage Object {}/{} has been created in bucket.", bucket, objectPath); - } } protected boolean doesBucketExist(final String bucket) { @@ -138,7 +133,18 @@ public String uploadRecordsToBucket(final SerializableBuffer recordsData, exceptionsThrown.add(e); } } - throw new RuntimeException(String.format("Exceptions thrown while uploading records into storage: %s", Strings.join(exceptionsThrown, "\n"))); + // Verifying that ALL exceptions are authentication related before assuming this is a configuration + // issue + // reduces risk of misidentifying errors or reporting a transient error. + final boolean areAllExceptionsAuthExceptions = exceptionsThrown.stream().filter(e -> e instanceof AmazonS3Exception) + .map(s3e -> ((AmazonS3Exception) s3e).getStatusCode()) + .filter(ConnectorExceptionUtil.HTTP_AUTHENTICATION_ERROR_CODES::contains) + .count() == exceptionsThrown.size(); + if (areAllExceptionsAuthExceptions) { + throw new ConfigErrorException(exceptionsThrown.get(0).getMessage(), exceptionsThrown.get(0)); + } else { + throw new RuntimeException(String.format("Exceptions thrown while uploading records into storage: %s", Strings.join(exceptionsThrown, "\n"))); + } } /** diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java index 2db68dff49e6..ea336d8fce65 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.util; +import com.google.common.collect.ImmutableList; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.integrations.base.errors.messages.ErrorMessage; @@ -22,6 +23,8 @@ public class ConnectorExceptionUtil { static final String RECOVERY_CONNECTION_ERROR_MESSAGE = "We're having issues syncing from a Postgres replica that is configured as a hot standby server. " + "Please see https://docs.airbyte.com/integrations/sources/postgres/#sync-data-from-postgres-hot-standby-server for options and workarounds"; + + public static final List HTTP_AUTHENTICATION_ERROR_CODES = ImmutableList.of(401, 403); private static final List> configErrorPredicates = List.of(getConfigErrorPredicate(), getConnectionErrorPredicate(), isRecoveryConnectionExceptionPredicate(), isUnknownColumnInFieldListException()); diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 1cfd9ee045d5..e6cd0710659c 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.11 +LABEL io.airbyte.version=1.2.12 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index f79d950d1d7f..5e0eba63c40d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.11 +LABEL io.airbyte.version=1.2.12 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index 51a269725bef..e955d005b871 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -13,11 +13,14 @@ import com.google.cloud.bigquery.LoadJobConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableId; +import com.google.common.collect.ImmutableList; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.GcsStorageOperations; import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.integrations.util.ConnectorExceptionUtil; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.HashSet; import java.util.List; @@ -84,7 +87,15 @@ public String getStagingFullPath(final String datasetId, final String stream) { public void createSchemaIfNotExists(final String datasetId, final String datasetLocation) { if (!existingSchemas.contains(datasetId)) { LOGGER.info("Creating dataset {}", datasetId); - BigQueryUtils.getOrCreateDataset(bigQuery, datasetId, datasetLocation); + try { + BigQueryUtils.getOrCreateDataset(bigQuery, datasetId, datasetLocation); + } catch (BigQueryException e) { + if (ConnectorExceptionUtil.HTTP_AUTHENTICATION_ERROR_CODES.contains(e.getCode())) { + throw new ConfigErrorException(e.getMessage(), e); + } else { + throw e; + } + } existingSchemas.add(datasetId); } } @@ -99,7 +110,7 @@ public void createTmpTableIfNotExists(final TableId tmpTableId, final Schema tab public void createStageIfNotExists(final String datasetId, final String stream) { final String objectPath = getStagingFullPath(datasetId, stream); LOGGER.info("Creating staging path for stream {} (dataset {}): {}", stream, datasetId, objectPath); - gcsStorageOperations.createBucketObjectIfNotExists(objectPath); + gcsStorageOperations.createBucketIfNotExists(); } @Override diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index 065ff49844ce..3f2c05210cf0 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -84,8 +84,10 @@ class BigQueryDestinationTest { protected static final Path CREDENTIALS_WITH_GCS_STAGING_PATH = Path.of("secrets/credentials-gcs-staging.json"); - protected static final Path[] ALL_PATHS = {CREDENTIALS_WITH_GCS_STAGING_PATH, CREDENTIALS_BAD_PROJECT_PATH, CREDENTIALS_NO_DATASET_CREATION_PATH, + + protected static final Path[] ALL_PATHS = {CREDENTIALS_STANDARD_INSERT_PATH, CREDENTIALS_BAD_PROJECT_PATH, CREDENTIALS_NO_DATASET_CREATION_PATH, CREDENTIALS_NO_EDIT_PUBLIC_SCHEMA_ROLE_PATH, CREDENTIALS_NON_BILLABLE_PROJECT_PATH, CREDENTIALS_WITH_GCS_STAGING_PATH}; + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestinationTest.class); private static final String DATASET_NAME_PREFIX = "bq_dest_integration_test"; diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile index c5534e0a9128..f4d34c04bceb 100644 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ b/airbyte-integrations/connectors/destination-gcs/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.12 +LABEL io.airbyte.version=0.2.13 LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 5f06bd32cc8e..882852e88811 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.53 +LABEL io.airbyte.version=0.3.54 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java index 63abde3f6966..657a582d6eeb 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java @@ -77,7 +77,7 @@ public String getStagingPath(UUID connectionId, String namespace, String streamN public void createStageIfNotExists(JdbcDatabase database, String stageName) throws Exception { final String bucketPath = s3Config.getBucketPath(); final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); - s3StorageOperations.createBucketObjectIfNotExists(prefix + stageName); + s3StorageOperations.createBucketIfNotExists(); } @Override diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index e11e5ea30b84..73f794c6b721 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -40,5 +40,5 @@ RUN /bin/bash -c 'set -e && \ echo "unknown arch" ;\ fi' -LABEL io.airbyte.version=0.3.18 +LABEL io.airbyte.version=0.3.19 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 4b8f627b28ea..da6515de1feb 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -20,5 +20,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=0.4.43 +LABEL io.airbyte.version=0.4.44 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java index 9cdc3058be4e..e9c40de09116 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java @@ -82,7 +82,7 @@ public String uploadRecordsToStage(final JdbcDatabase database, @Override public void createStageIfNotExists(final JdbcDatabase database, final String stageName) { - s3StorageOperations.createBucketObjectIfNotExists(stageName); + s3StorageOperations.createBucketIfNotExists(); } @Override diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index e418918e1f78..6b6759b152f5 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -136,6 +136,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| +| 1.2.12 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 1.2.11 | 2023-01-18 | [#21144](https://github.com/airbytehq/airbyte/pull/21144) | Added explicit error message if sync fails due to a config issue | | 1.2.9 | 2022-12-14 | [#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check | | 1.2.8 | 2022-11-22 | [#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index aa07d098ed17..e988530da08c 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -235,31 +235,32 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.2.12 | 2022-10-18 | [\#17901](https://github.com/airbytehq/airbyte/pull/17901) | Fix logging to GCS | -| 0.2.11 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | -| 0.2.10 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 0.2.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | -| 0.2.8 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | -| 0.2.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | -| 0.2.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | -| 0.2.5 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | -| 0.2.4 | 2022-04-22 | [\#12167](https://github.com/airbytehq/airbyte/pull/12167) | Add gzip compression option for CSV and JSONL formats. | -| 0.2.3 | 2022-04-22 | [\#11795](https://github.com/airbytehq/airbyte/pull/11795) | Fix the connection check to verify the provided bucket path. | -| 0.2.2 | 2022-04-05 | [\#11728](https://github.com/airbytehq/airbyte/pull/11728) | Properly clean-up bucket when running OVERWRITE sync mode | -| 0.2.1 | 2022-04-05 | [\#11499](https://github.com/airbytehq/airbyte/pull/11499) | Updated spec and documentation. | -| 0.2.0 | 2022-04-04 | [\#11686](https://github.com/airbytehq/airbyte/pull/11686) | Use serialized buffering strategy to reduce memory consumption; compress CSV and JSONL formats. | -| 0.1.22 | 2022-02-12 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add JVM flag to exist on OOME. | -| 0.1.21 | 2022-02-12 | [\#10299](https://github.com/airbytehq/airbyte/pull/10299) | Fix connection check to require only the necessary permissions. | -| 0.1.20 | 2022-01-11 | [\#9367](https://github.com/airbytehq/airbyte/pull/9367) | Avro & Parquet: support array field with unknown item type; default any improperly typed field to string. | -| 0.1.19 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | -| 0.1.18 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | -| 0.1.17 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types | -| 0.1.16 | 2021-12-20 | [\#8974](https://github.com/airbytehq/airbyte/pull/8974) | Release a new version to ensure there is no excessive logging. | -| 0.1.15 | 2021-12-03 | [\#8386](https://github.com/airbytehq/airbyte/pull/8386) | Add new GCP regions | -| 0.1.14 | 2021-12-01 | [\#7732](https://github.com/airbytehq/airbyte/pull/7732) | Support timestamp in Avro and Parquet | -| 0.1.13 | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288) | Support Json `additionalProperties`. | -| 0.1.2 | 2021-09-12 | [\#5720](https://github.com/airbytehq/airbyte/issues/5720) | Added configurable block size for stream. Each stream is limited to 10,000 by GCS | -| 0.1.1 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added storing gcsCsvFileLocation property for CSV format. This is used by destination-bigquery \(GCS Staging upload type\) | -| 0.1.0 | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784) | Initial release. | +| Version | Date | Pull Request | Subject | +|:--------| :--- |:------------------------------------------------------------| :--- | +| 0.2.13 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | +| 0.2.12 | 2022-10-18 | [\#17901](https://github.com/airbytehq/airbyte/pull/17901) | Fix logging to GCS | +| 0.2.11 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | +| 0.2.10 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.2.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | +| 0.2.8 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | +| 0.2.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | +| 0.2.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | +| 0.2.5 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | +| 0.2.4 | 2022-04-22 | [\#12167](https://github.com/airbytehq/airbyte/pull/12167) | Add gzip compression option for CSV and JSONL formats. | +| 0.2.3 | 2022-04-22 | [\#11795](https://github.com/airbytehq/airbyte/pull/11795) | Fix the connection check to verify the provided bucket path. | +| 0.2.2 | 2022-04-05 | [\#11728](https://github.com/airbytehq/airbyte/pull/11728) | Properly clean-up bucket when running OVERWRITE sync mode | +| 0.2.1 | 2022-04-05 | [\#11499](https://github.com/airbytehq/airbyte/pull/11499) | Updated spec and documentation. | +| 0.2.0 | 2022-04-04 | [\#11686](https://github.com/airbytehq/airbyte/pull/11686) | Use serialized buffering strategy to reduce memory consumption; compress CSV and JSONL formats. | +| 0.1.22 | 2022-02-12 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add JVM flag to exist on OOME. | +| 0.1.21 | 2022-02-12 | [\#10299](https://github.com/airbytehq/airbyte/pull/10299) | Fix connection check to require only the necessary permissions. | +| 0.1.20 | 2022-01-11 | [\#9367](https://github.com/airbytehq/airbyte/pull/9367) | Avro & Parquet: support array field with unknown item type; default any improperly typed field to string. | +| 0.1.19 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | +| 0.1.18 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | +| 0.1.17 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types | +| 0.1.16 | 2021-12-20 | [\#8974](https://github.com/airbytehq/airbyte/pull/8974) | Release a new version to ensure there is no excessive logging. | +| 0.1.15 | 2021-12-03 | [\#8386](https://github.com/airbytehq/airbyte/pull/8386) | Add new GCP regions | +| 0.1.14 | 2021-12-01 | [\#7732](https://github.com/airbytehq/airbyte/pull/7732) | Support timestamp in Avro and Parquet | +| 0.1.13 | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288) | Support Json `additionalProperties`. | +| 0.1.2 | 2021-09-12 | [\#5720](https://github.com/airbytehq/airbyte/issues/5720) | Added configurable block size for stream. Each stream is limited to 10,000 by GCS | +| 0.1.1 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added storing gcsCsvFileLocation property for CSV format. This is used by destination-bigquery \(GCS Staging upload type\) | +| 0.1.0 | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784) | Initial release. | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index a2a7168dcd03..0f2b0ac8e50f 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -141,6 +141,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.54 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 0.3.53 | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field | | 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers | | 0.3.51 | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling | @@ -178,5 +179,3 @@ Each stream will be output into its own raw table in Redshift. Each table will c | 0.3.13 | 2021-09-02 | [5745](https://github.com/airbytehq/airbyte/pull/5745) | Disable STATUPDATE flag when using S3 staging to speed up performance | | 0.3.12 | 2021-07-21 | [3555](https://github.com/airbytehq/airbyte/pull/3555) | Enable partial checkpointing for halfway syncs | | 0.3.11 | 2021-07-20 | [4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | - - diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index c642624c6ea3..db26ac102e0d 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -330,6 +330,7 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.19 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 0.3.18 | 2022-12-15 | [\#20088](https://github.com/airbytehq/airbyte/pull/20088) | New data type support v0/v1 | | 0.3.17 | 2022-10-15 | [\#18031](https://github.com/airbytehq/airbyte/pull/18031) | Fix integration tests to use bucket path | | 0.3.16 | 2022-10-03 | [\#17340](https://github.com/airbytehq/airbyte/pull/17340) | Enforced encrypted only traffic to S3 buckets and check logic | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index b9c6ba290ce2..1bbadee5053a 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -277,7 +277,8 @@ Now that you have set up the Snowflake destination connector, check out the foll | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.43 | 2023-01-20 | [\#21450](https://github.com/airbytehq/airbyte/pull/21450) | Updated Check methods to handle more possible s3 and gcs stagings issues | +| 0.4.44 | 2023-01-20 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | +| 0.4.43 | 2023-01-20 | [\#21450](https://github.com/airbytehq/airbyte/pull/21450) | Updated Check methods to handle more possible s3 and gcs stagings issues | | 0.4.42 | 2023-01-12 | [\#21342](https://github.com/airbytehq/airbyte/pull/21342) | Better handling for conflicting destination streams | | 0.4.41 | 2022-12-16 | [\#20566](https://github.com/airbytehq/airbyte/pull/20566) | Improve spec to adhere to standards | | 0.4.40 | 2022-11-11 | [\#19302](https://github.com/airbytehq/airbyte/pull/19302) | Set jdbc application env variable depends on env - airbyte_oss or airbyte_cloud | From d77514dcfedbc4205f7f6161f29200a32d5d06d1 Mon Sep 17 00:00:00 2001 From: Augustin Date: Mon, 23 Jan 2023 18:25:53 +0100 Subject: [PATCH 09/11] CI connector ops: introduce QA checks in /test (#21699) --- .github/workflows/test-command.yml | 9 -- tools/bin/ci_integration_test.sh | 1 + .../ci_connector_ops/qa_checks.py | 91 +++++++++++++++++++ tools/ci_connector_ops/setup.py | 5 +- 4 files changed, 95 insertions(+), 11 deletions(-) create mode 100644 tools/ci_connector_ops/ci_connector_ops/qa_checks.py diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 069d1cd46624..8a6f36d5d775 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -95,19 +95,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.9" - - name: Install Pyenv - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate - name: Install CI scripts # all CI python packages have the prefix "ci_" run: | - source venv/bin/activate pip install --quiet -e ./tools/ci_* - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} run: | - source venv/bin/activate ci_credentials ${{ github.event.inputs.connector }} write-to-storage # normalization also runs destination-specific tests, so fetch their creds also if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then @@ -117,7 +110,6 @@ jobs: fi env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Test ${{ github.event.inputs.connector }} id: test env: @@ -132,7 +124,6 @@ jobs: - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} if: always() run: | - source venv/bin/activate ci_credentials ${{ github.event.inputs.connector }} update-secrets # normalization also runs destination-specific tests, so fetch their creds also if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then diff --git a/tools/bin/ci_integration_test.sh b/tools/bin/ci_integration_test.sh index 8fb208fdf83a..3fbd1a65970d 100755 --- a/tools/bin/ci_integration_test.sh +++ b/tools/bin/ci_integration_test.sh @@ -32,6 +32,7 @@ else export SUB_BUILD="CONNECTORS_BASE" elif [[ "$connector" == *"connectors"* ]]; then connector_name=$(echo $connector | cut -d / -f 2) + run-qa-checks $connector_name selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector_name$" || echo "") integrationTestCommand="$(_to_gradle_path "airbyte-integrations/$connector" integrationTest)" else diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_checks.py b/tools/ci_connector_ops/ci_connector_ops/qa_checks.py new file mode 100644 index 000000000000..dc384641c454 --- /dev/null +++ b/tools/ci_connector_ops/ci_connector_ops/qa_checks.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys + +def check_documentation_markdown_file(connector_name: str) -> bool: + """Check if a markdown file with connector documentation is available + in docs/integrations//.md + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether a documentation file was found. + """ + # TODO implement + return True + +def check_changelog_entry_is_updated(connector_name: str) -> bool: + """Check that the changelog entry is updated for the latest connector version + in docs/integrations//.md + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether a the changelog is up to date. + """ + # TODO implement + return True + +def check_connector_icon_is_available(connector_name: str) -> bool: + """Check an SVG icon exists for a connector in + in airbyte-config/init/src/main/resources/icons/.svg + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether an icon exists for this connector. + """ + # TODO implement + return True + +def check_connector_https_url_only(connector_name: str) -> bool: + """Check a connector code contains only https url. + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether the connector code contains only https url. + """ + # TODO implement + return True + +def check_connector_has_no_critical_vulnerabilities(connector_name: str) -> bool: + """Check if the connector image is free of critical Snyk vulnerabilities. + Runs a docker scan command. + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether the connector is free of critical vulnerabilities. + """ + # TODO implement + return True + +QA_CHECKS = [ + check_documentation_markdown_file, + check_changelog_entry_is_updated, + check_connector_icon_is_available, + check_connector_https_url_only, + check_connector_has_no_critical_vulnerabilities +] + +def run_qa_checks(): + connector_name = sys.argv[1] + print(f"Running QA checks for {connector_name}") + qa_check_results = {qa_check.__name__: qa_check(connector_name) for qa_check in QA_CHECKS} + if not all(qa_check_results.values()): + print(f"QA checks failed for {connector_name}") + for check_name, check_result in qa_check_results.items(): + check_result_prefix = "✅" if check_result else "❌" + print(f"{check_result_prefix} - {check_name}") + sys.exit(1) + else: + print(f"All QA checks succeeded for {connector_name}") + sys.exit(0) diff --git a/tools/ci_connector_ops/setup.py b/tools/ci_connector_ops/setup.py index 09928f800c24..c573c8d7148b 100644 --- a/tools/ci_connector_ops/setup.py +++ b/tools/ci_connector_ops/setup.py @@ -9,7 +9,7 @@ setup( - version="0.1.1", + version="0.1.2", name="ci_connector_ops", description="Packaged maintained by the connector operations team to perform CI for connectors", author="Airbyte", @@ -21,7 +21,8 @@ "console_scripts": [ "check-test-strictness-level = ci_connector_ops.sat_config_checks:check_test_strictness_level", "write-review-requirements-file = ci_connector_ops.sat_config_checks:write_review_requirements_file", - "print-mandatory-reviewers = ci_connector_ops.sat_config_checks:print_mandatory_reviewers" + "print-mandatory-reviewers = ci_connector_ops.sat_config_checks:print_mandatory_reviewers", + "run-qa-checks = ci_connector_ops.qa_checks:run_qa_checks" ], }, ) From b786a180f02c6d48d82f19f51c8c8e730a14b983 Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin Date: Mon, 23 Jan 2023 12:42:43 -0500 Subject: [PATCH 10/11] Add required roles to API endpoints (#21664) * Restore auth poc * Formatting * Custom Netty pipeline handler to aid authorization * Fix handler name * Cleanup * Remove cloud code * Disable API authorization in OSS * Remove unused dependency * Add newline * Add required roles --- .../io/airbyte/commons/auth/AuthRole.java | 92 +++++++++++++++++++ .../commons/auth/AuthRoleConstants.java | 21 +++++ .../io/airbyte/commons/auth/AuthRoleTest.java | 47 ++++++++++ .../server/apis/AttemptApiController.java | 6 ++ .../server/apis/ConnectionApiController.java | 14 +++ .../server/apis/DestinationApiController.java | 20 +++- .../DestinationDefinitionApiController.java | 19 ++++ ...nDefinitionSpecificationApiController.java | 6 ++ .../apis/DestinationOauthApiController.java | 9 ++ .../server/apis/HealthApiController.java | 3 + .../server/apis/JobsApiController.java | 14 +++ .../server/apis/LogsApiController.java | 6 ++ .../server/apis/NotFoundController.java | 3 + .../apis/NotificationsApiController.java | 6 ++ .../server/apis/OpenapiApiController.java | 6 ++ .../server/apis/OperationApiController.java | 13 +++ .../server/apis/SchedulerApiController.java | 9 ++ .../server/apis/SourceApiController.java | 15 +++ .../apis/SourceDefinitionApiController.java | 19 ++++ ...eDefinitionSpecificationApiController.java | 6 ++ .../server/apis/SourceOauthApiController.java | 9 ++ .../server/apis/StateApiController.java | 8 ++ .../server/apis/WebBackendApiController.java | 15 +++ .../server/apis/WorkspaceApiController.java | 16 ++++ .../src/main/resources/application.yml | 48 +++++++--- deps.toml | 2 + 26 files changed, 417 insertions(+), 15 deletions(-) create mode 100644 airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRole.java create mode 100644 airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java create mode 100644 airbyte-commons/src/test/java/io/airbyte/commons/auth/AuthRoleTest.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRole.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRole.java new file mode 100644 index 000000000000..b2ff02702545 --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRole.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This enum describes the standard auth levels for a given resource. It currently is only used for + * 2 resources Workspace and Instance (i.e. the entire instance or deployment of Airbyte). + * + * In the context of a workspace, there is a 1:1 mapping. + *
    + *
  • OWNER => WORKSPACE OWNER. Superadmin of the instance (typically the person that created it), + * has all the rights on the instance including deleting it.
    • + *
  • ADMIN => WORKSPACE ADMIN. Admin of the instance, can invite other users, update their + * permission and change settings of the instance.
    • + *
  • EDITOR => WORKSPACE EDITOR
    • + *
  • READER => WORKSPACE READER
    • + *
  • AUTHENTICATED_USER => INVALID
    • + *
  • NONE => NONE (does not have access to this resource)
    • + *
+ * In the context of the instance, there are currently only 3 levels. + *
    + *
  • ADMIN => INSTANCE ADMIN
    • + *
  • AUTHENTICATED_USER => Denotes that all that is required for access is an active Airbyte + * account. This should only ever be used when the associated resource is an INSTANCE. All other + * uses are invalid. It is a special value in the enum to handle a case that only applies to + * instances and no other resources.
    • + *
  • NONE => NONE (not applicable. anyone being checked in our auth stack already has an account + * so by definition they have some access to the instance.)
    • + *
+ */ +public enum AuthRole { + + OWNER(500, AuthRoleConstants.OWNER), + ADMIN(400, AuthRoleConstants.ADMIN), + EDITOR(300, AuthRoleConstants.EDITOR), + READER(200, AuthRoleConstants.READER), + AUTHENTICATED_USER(100, AuthRoleConstants.AUTHENTICATED_USER), // ONLY USE WITH INSTANCE RESOURCE! + NONE(0, AuthRoleConstants.NONE); + + private final int authority; + private final String label; + + AuthRole(final int authority, final String label) { + this.authority = authority; + this.label = label; + } + + public int getAuthority() { + return authority; + } + + public String getLabel() { + return label; + } + + /** + * Builds the set of roles based on the provided {@link AuthRole} value. + *

+ * The generated set of auth roles contains the provided {@link AuthRole} (if not {@code null}) and + * any other authentication roles with a lesser {@link #getAuthority()} value. + *

+ * + * @param authRole An {@link AuthRole} (may be {@code null}). + * @return The set of {@link AuthRole}s based on the provided {@link AuthRole}. + */ + public static Set buildAuthRolesSet(final AuthRole authRole) { + final Set authRoles = new HashSet<>(); + + if (authRole != null) { + authRoles.add(authRole); + authRoles.addAll(Stream.of(values()) + .filter(role -> !NONE.equals(role)) + .filter(role -> role.getAuthority() < authRole.getAuthority()) + .collect(Collectors.toSet())); + } + + // Sort final set by descending authority order + return authRoles.stream() + .sorted(Comparator.comparingInt(AuthRole::getAuthority)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java new file mode 100644 index 000000000000..6a206ee0e89e --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +/** + * Collection of constants that defines authorization roles. + */ +public final class AuthRoleConstants { + + public static final String ADMIN = "ADMIN"; + public static final String AUTHENTICATED_USER = "AUTHENTICATED_USER"; + public static final String EDITOR = "EDITOR"; + public static final String OWNER = "OWNER"; + public static final String NONE = "NONE"; + public static final String READER = "READER"; + + private AuthRoleConstants() {} + +} diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/auth/AuthRoleTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/auth/AuthRoleTest.java new file mode 100644 index 000000000000..835488fdd84a --- /dev/null +++ b/airbyte-commons/src/test/java/io/airbyte/commons/auth/AuthRoleTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Set; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link AuthRole} enumeration. + */ +class AuthRoleTest { + + @Test + void testBuildingAuthRoleSet() { + final Set ownerResult = AuthRole.buildAuthRolesSet(AuthRole.OWNER); + assertEquals(5, ownerResult.size()); + assertEquals(Set.of(AuthRole.OWNER, AuthRole.ADMIN, AuthRole.EDITOR, AuthRole.READER, AuthRole.AUTHENTICATED_USER), ownerResult); + + final Set adminResult = AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + assertEquals(4, adminResult.size()); + assertEquals(Set.of(AuthRole.ADMIN, AuthRole.EDITOR, AuthRole.READER, AuthRole.AUTHENTICATED_USER), adminResult); + + final Set editorResult = AuthRole.buildAuthRolesSet(AuthRole.EDITOR); + assertEquals(3, editorResult.size()); + assertEquals(Set.of(AuthRole.EDITOR, AuthRole.READER, AuthRole.AUTHENTICATED_USER), editorResult); + + final Set readerResult = AuthRole.buildAuthRolesSet(AuthRole.READER); + assertEquals(2, readerResult.size()); + assertEquals(Set.of(AuthRole.READER, AuthRole.AUTHENTICATED_USER), readerResult); + + final Set authenticatedUserResult = AuthRole.buildAuthRolesSet(AuthRole.AUTHENTICATED_USER); + assertEquals(1, authenticatedUserResult.size()); + assertEquals(Set.of(AuthRole.AUTHENTICATED_USER), authenticatedUserResult); + + final Set noneResult = AuthRole.buildAuthRolesSet(AuthRole.NONE); + assertEquals(1, noneResult.size()); + assertEquals(Set.of(AuthRole.NONE), noneResult); + + final Set nullResult = AuthRole.buildAuthRolesSet(null); + assertEquals(0, nullResult.size()); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java index d50e731d5e8d..c7c112405b44 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; + import io.airbyte.api.generated.AttemptApi; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.SaveStatsRequestBody; @@ -14,10 +16,13 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/attempt/") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class AttemptApiController implements AttemptApi { private final AttemptHandler attemptHandler; @@ -36,6 +41,7 @@ public InternalOperationResult saveStats(final SaveStatsRequestBody requestBody) @Override @Post(uri = "/set_workflow_in_attempt", processes = MediaType.APPLICATION_JSON) + @Secured({ADMIN}) public InternalOperationResult setWorkflowInAttempt(@Body final SetWorkflowInAttemptRequestBody requestBody) { return ApiHelper.execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index f0de78d03d31..f61b89e41447 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.ConnectionApi; import io.airbyte.api.model.generated.ConnectionCreate; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -21,11 +24,14 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/connections") @Context() @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class ConnectionApiController implements ConnectionApi { private final ConnectionsHandler connectionsHandler; @@ -42,24 +48,28 @@ public ConnectionApiController(final ConnectionsHandler connectionsHandler, @Override @Post(uri = "/create") + @Secured({EDITOR}) public ConnectionRead createConnection(@Body final ConnectionCreate connectionCreate) { return ApiHelper.execute(() -> connectionsHandler.createConnection(connectionCreate)); } @Override @Post(uri = "/update") + @Secured({EDITOR}) public ConnectionRead updateConnection(@Body final ConnectionUpdate connectionUpdate) { return ApiHelper.execute(() -> connectionsHandler.updateConnection(connectionUpdate)); } @Override @Post(uri = "/list") + @Secured({READER}) public ConnectionReadList listConnectionsForWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)); } @Override @Post(uri = "/list_all") + @Secured({READER}) public ConnectionReadList listAllConnectionsForWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> connectionsHandler.listAllConnectionsForWorkspace(workspaceIdRequestBody)); } @@ -72,12 +82,14 @@ public ConnectionReadList searchConnections(@Body final ConnectionSearch connect @Override @Post(uri = "/get") + @Secured({READER}) public ConnectionRead getConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> connectionsHandler.getConnection(connectionIdRequestBody.getConnectionId())); } @Override @Post(uri = "/delete") + @Secured({EDITOR}) public void deleteConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { ApiHelper.execute(() -> { operationsHandler.deleteOperationsForConnection(connectionIdRequestBody); @@ -88,12 +100,14 @@ public void deleteConnection(@Body final ConnectionIdRequestBody connectionIdReq @Override @Post(uri = "/sync") + @Secured({EDITOR}) public JobInfoRead syncConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.syncConnection(connectionIdRequestBody)); } @Override @Post(uri = "/reset") + @Secured({EDITOR}) public JobInfoRead resetConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.resetConnection(connectionIdRequestBody)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java index 6c16cec4a596..7a20184b0e4d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.DestinationApi; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.DestinationCloneRequestBody; @@ -20,24 +23,32 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; -import lombok.AllArgsConstructor; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destinations") @Requires(property = "airbyte.deployment-mode", value = "OSS") -@AllArgsConstructor +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationApiController implements DestinationApi { private final DestinationHandler destinationHandler; private final SchedulerHandler schedulerHandler; + public DestinationApiController(final DestinationHandler destinationHandler, final SchedulerHandler schedulerHandler) { + this.destinationHandler = destinationHandler; + this.schedulerHandler = schedulerHandler; + } + @Post(uri = "/check_connection") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToDestination(@Body final DestinationIdRequestBody destinationIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationId(destinationIdRequestBody)); } @Post(uri = "/check_connection_for_update") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToDestinationForUpdate(@Body final DestinationUpdate destinationUpdate) { return ApiHelper.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationIdForUpdate(destinationUpdate)); @@ -50,12 +61,14 @@ public DestinationRead cloneDestination(@Body final DestinationCloneRequestBody } @Post(uri = "/create") + @Secured({EDITOR}) @Override public DestinationRead createDestination(@Body final DestinationCreate destinationCreate) { return ApiHelper.execute(() -> destinationHandler.createDestination(destinationCreate)); } @Post(uri = "/delete") + @Secured({EDITOR}) @Override public void deleteDestination(@Body final DestinationIdRequestBody destinationIdRequestBody) { ApiHelper.execute(() -> { @@ -65,12 +78,14 @@ public void deleteDestination(@Body final DestinationIdRequestBody destinationId } @Post(uri = "/get") + @Secured({READER}) @Override public DestinationRead getDestination(@Body final DestinationIdRequestBody destinationIdRequestBody) { return ApiHelper.execute(() -> destinationHandler.getDestination(destinationIdRequestBody)); } @Post(uri = "/list") + @Secured({READER}) @Override public DestinationReadList listDestinationsForWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> destinationHandler.listDestinationsForWorkspace(workspaceIdRequestBody)); @@ -83,6 +98,7 @@ public DestinationReadList searchDestinations(@Body final DestinationSearch dest } @Post(uri = "/update") + @Secured({EDITOR}) @Override public DestinationRead updateDestination(@Body final DestinationUpdate destinationUpdate) { return ApiHelper.execute(() -> destinationHandler.updateDestination(destinationUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java index 54409d281021..ab559f6b3b09 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java @@ -4,6 +4,11 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.DestinationDefinitionApi; import io.airbyte.api.model.generated.CustomDestinationDefinitionCreate; import io.airbyte.api.model.generated.DestinationDefinitionIdRequestBody; @@ -19,11 +24,14 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destination_definitions") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationDefinitionApiController implements DestinationDefinitionApi { private final DestinationDefinitionsHandler destinationDefinitionsHandler; @@ -33,12 +41,14 @@ public DestinationDefinitionApiController(final DestinationDefinitionsHandler de } @Post(uri = "/create_custom") + @Secured({EDITOR}) @Override public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { return ApiHelper.execute(() -> destinationDefinitionsHandler.createCustomDestinationDefinition(customDestinationDefinitionCreate)); } @Post(uri = "/delete") + @Secured({ADMIN}) @Override public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { ApiHelper.execute(() -> { @@ -48,18 +58,21 @@ public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody } @Post(uri = "/get") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { return ApiHelper.execute(() -> destinationDefinitionsHandler.getDestinationDefinition(destinationDefinitionIdRequestBody)); } @Post(uri = "/get_for_workspace") + @Secured({READER}) @Override public DestinationDefinitionRead getDestinationDefinitionForWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> destinationDefinitionsHandler.getDestinationDefinitionForWorkspace(destinationDefinitionIdWithWorkspaceId)); } @Post(uri = "/grant_definition") + @Secured({ADMIN}) @Override public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { return ApiHelper @@ -67,30 +80,35 @@ public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace(fi } @Post(uri = "/list") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionReadList listDestinationDefinitions() { return ApiHelper.execute(destinationDefinitionsHandler::listDestinationDefinitions); } @Post(uri = "/list_for_workspace") + @Secured({READER}) @Override public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> destinationDefinitionsHandler.listDestinationDefinitionsForWorkspace(workspaceIdRequestBody)); } @Post(uri = "/list_latest") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionReadList listLatestDestinationDefinitions() { return ApiHelper.execute(destinationDefinitionsHandler::listLatestDestinationDefinitions); } @Post(uri = "/list_private") + @Secured({ADMIN}) @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); } @Post(uri = "/revoke_definition") + @Secured({ADMIN}) @Override public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { ApiHelper.execute(() -> { @@ -100,6 +118,7 @@ public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinition } @Post(uri = "/update") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { return ApiHelper.execute(() -> destinationDefinitionsHandler.updateDestinationDefinition(destinationDefinitionUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java index 54492c4a905f..d93bc6dcbe99 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.DestinationDefinitionSpecificationApi; import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; @@ -11,10 +13,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destination_definition_specifications") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationDefinitionSpecificationApiController implements DestinationDefinitionSpecificationApi { private final SchedulerHandler schedulerHandler; @@ -24,6 +29,7 @@ public DestinationDefinitionSpecificationApiController(final SchedulerHandler sc } @Post("/get") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java index 33318e14efcb..58286dfacd48 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; + import io.airbyte.api.generated.DestinationOauthApi; import io.airbyte.api.model.generated.CompleteDestinationOAuthRequest; import io.airbyte.api.model.generated.DestinationOauthConsentRequest; @@ -14,12 +17,15 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.util.Map; @Controller("/api/v1/destination_oauths") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationOauthApiController implements DestinationOauthApi { private final OAuthHandler oAuthHandler; @@ -29,18 +35,21 @@ public DestinationOauthApiController(final OAuthHandler oAuthHandler) { } @Post("/complete_oauth") + @Secured({EDITOR}) @Override public Map completeDestinationOAuth(final CompleteDestinationOAuthRequest completeDestinationOAuthRequest) { return ApiHelper.execute(() -> oAuthHandler.completeDestinationOAuth(completeDestinationOAuthRequest)); } @Post("/get_consent_url") + @Secured({EDITOR}) @Override public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsentRequest destinationOauthConsentRequest) { return ApiHelper.execute(() -> oAuthHandler.getDestinationOAuthConsent(destinationOauthConsentRequest)); } @Post("/oauth_params/create") + @Secured({ADMIN}) @Override public void setInstancewideDestinationOauthParams(final SetInstancewideDestinationOauthParamsRequestBody setInstancewideDestinationOauthParamsRequestBody) { ApiHelper.execute(() -> { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java index a5b46d39742b..3ffb6851cbc5 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java @@ -11,10 +11,13 @@ import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/health") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_ANONYMOUS) public class HealthApiController implements HealthApi { private final HealthCheckHandler healthCheckHandler; diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java index 9fadce045d05..b930472e5d23 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java @@ -4,6 +4,10 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.JobsApi; import io.airbyte.api.model.generated.AttemptNormalizationStatusReadList; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -20,11 +24,14 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/jobs") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class JobsApiController implements JobsApi { private final JobHistoryHandler jobHistoryHandler; @@ -36,42 +43,49 @@ public JobsApiController(final JobHistoryHandler jobHistoryHandler, final Schedu } @Post("/cancel") + @Secured({EDITOR}) @Override public JobInfoRead cancelJob(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.cancelJob(jobIdRequestBody)); } @Post("/get_normalization_status") + @Secured({ADMIN}) @Override public AttemptNormalizationStatusReadList getAttemptNormalizationStatusesForJob(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getAttemptNormalizationStatuses(jobIdRequestBody)); } @Post("/get_debug_info") + @Secured({READER}) @Override public JobDebugInfoRead getJobDebugInfo(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getJobDebugInfo(jobIdRequestBody)); } @Post("/get") + @Secured({READER}) @Override public JobInfoRead getJobInfo(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getJobInfo(jobIdRequestBody)); } @Post("/get_light") + @Secured({READER}) @Override public JobInfoLightRead getJobInfoLight(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getJobInfoLight(jobIdRequestBody)); } @Post("/get_last_replication_job") + @Secured({READER}) @Override public JobOptionalRead getLastReplicationJob(final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getLastReplicationJob(connectionIdRequestBody)); } @Post("/list") + @Secured({READER}) @Override public JobReadList listJobsFor(final JobListRequestBody jobListRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.listJobsFor(jobListRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java index 126ed8202693..13257412cf04 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; + import io.airbyte.api.generated.LogsApi; import io.airbyte.api.model.generated.LogsRequestBody; import io.airbyte.server.handlers.LogsHandler; @@ -11,12 +13,15 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.io.File; @Controller("/api/v1/logs") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class LogsApiController implements LogsApi { private final LogsHandler logsHandler; @@ -26,6 +31,7 @@ public LogsApiController(final LogsHandler logsHandler) { } @Post("/get") + @Secured({ADMIN}) @Override public File getLogs(final LogsRequestBody logsRequestBody) { return ApiHelper.execute(() -> logsHandler.getLogs(logsRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java index a59dea440b05..0a089a21b762 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java @@ -11,6 +11,8 @@ import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Error; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; /** * Custom controller that handles global 404 responses for unknown/unmapped paths. @@ -18,6 +20,7 @@ @Controller("/api/notfound") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_ANONYMOUS) public class NotFoundController { @Error(status = HttpStatus.NOT_FOUND, diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java index 6af6a1cdf242..c81799b9ac92 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.NotificationsApi; import io.airbyte.api.model.generated.Notification; import io.airbyte.api.model.generated.NotificationRead; @@ -12,10 +14,13 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/notifications/try") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class NotificationsApiController implements NotificationsApi { private final WorkspacesHandler workspacesHandler; @@ -25,6 +30,7 @@ public NotificationsApiController(final WorkspacesHandler workspacesHandler) { } @Post + @Secured({AUTHENTICATED_USER}) @Override public NotificationRead tryNotificationConfig(@Body final Notification notification) { return ApiHelper.execute(() -> workspacesHandler.tryNotification(notification)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java index 3aa984d7b2c7..5e574a23deb9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java @@ -4,16 +4,21 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.OpenapiApi; import io.airbyte.server.handlers.OpenApiConfigHandler; import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.io.File; @Controller("/api/v1/openapi") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class OpenapiApiController implements OpenapiApi { private final OpenApiConfigHandler openApiConfigHandler; @@ -23,6 +28,7 @@ public OpenapiApiController(final OpenApiConfigHandler openApiConfigHandler) { } @Get(produces = "text/plain") + @Secured({AUTHENTICATED_USER}) @Override public File getOpenApiSpec() { return ApiHelper.execute(openApiConfigHandler::getFile); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java index a892ca03d47b..172fbb76e40a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java @@ -4,6 +4,10 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.OperationApi; import io.airbyte.api.model.generated.CheckOperationRead; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -18,10 +22,13 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/operations") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class OperationApiController implements OperationApi { private final OperationsHandler operationsHandler; @@ -31,6 +38,7 @@ public OperationApiController(final OperationsHandler operationsHandler) { } @Post("/check") + @Secured({AUTHENTICATED_USER}) @Override public CheckOperationRead checkOperation(@Body final OperatorConfiguration operatorConfiguration) { return ApiHelper.execute(() -> operationsHandler.checkOperation(operatorConfiguration)); @@ -38,11 +46,13 @@ public CheckOperationRead checkOperation(@Body final OperatorConfiguration opera @Post("/create") @Override + @Secured({EDITOR}) public OperationRead createOperation(@Body final OperationCreate operationCreate) { return ApiHelper.execute(() -> operationsHandler.createOperation(operationCreate)); } @Post("/delete") + @Secured({EDITOR}) @Override public void deleteOperation(@Body final OperationIdRequestBody operationIdRequestBody) { ApiHelper.execute(() -> { @@ -52,18 +62,21 @@ public void deleteOperation(@Body final OperationIdRequestBody operationIdReques } @Post("/get") + @Secured({READER}) @Override public OperationRead getOperation(@Body final OperationIdRequestBody operationIdRequestBody) { return ApiHelper.execute(() -> operationsHandler.getOperation(operationIdRequestBody)); } @Post("/list") + @Secured({READER}) @Override public OperationReadList listOperationsForConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> operationsHandler.listOperationsForConnection(connectionIdRequestBody)); } @Post("/update") + @Secured({EDITOR}) @Override public OperationRead updateOperation(@Body final OperationUpdate operationUpdate) { return ApiHelper.execute(() -> operationsHandler.updateOperation(operationUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java index 53019774aeca..365dc04afc46 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; + import io.airbyte.api.generated.SchedulerApi; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.DestinationCoreConfig; @@ -13,10 +16,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/scheduler") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SchedulerApiController implements SchedulerApi { private final SchedulerHandler schedulerHandler; @@ -26,18 +32,21 @@ public SchedulerApiController(final SchedulerHandler schedulerHandler) { } @Post("/destinations/check_connection") + @Secured({AUTHENTICATED_USER}) @Override public CheckConnectionRead executeDestinationCheckConnection(final DestinationCoreConfig destinationCoreConfig) { return ApiHelper.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationCreate(destinationCoreConfig)); } @Post("/sources/check_connection") + @Secured({AUTHENTICATED_USER}) @Override public CheckConnectionRead executeSourceCheckConnection(final SourceCoreConfig sourceCoreConfig) { return ApiHelper.execute(() -> schedulerHandler.checkSourceConnectionFromSourceCreate(sourceCoreConfig)); } @Post("/sources/discover_schema") + @Secured({EDITOR}) @Override public SourceDiscoverSchemaRead executeSourceDiscoverSchema(final SourceCoreConfig sourceCoreConfig) { return ApiHelper.execute(() -> schedulerHandler.discoverSchemaForSourceFromSourceCreate(sourceCoreConfig)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java index 64cf7c58dfd6..0c680f742989 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.SourceApi; import io.airbyte.api.model.generated.ActorCatalogWithUpdatedAt; import io.airbyte.api.model.generated.CheckConnectionRead; @@ -22,10 +25,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/sources") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceApiController implements SourceApi { private final SchedulerHandler schedulerHandler; @@ -37,12 +43,14 @@ public SourceApiController(final SchedulerHandler schedulerHandler, final Source } @Post("/check_connection") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToSource(final SourceIdRequestBody sourceIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.checkSourceConnectionFromSourceId(sourceIdRequestBody)); } @Post("/check_connection_for_update") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToSourceForUpdate(final SourceUpdate sourceUpdate) { return ApiHelper.execute(() -> schedulerHandler.checkSourceConnectionFromSourceIdForUpdate(sourceUpdate)); @@ -55,12 +63,14 @@ public SourceRead cloneSource(final SourceCloneRequestBody sourceCloneRequestBod } @Post("/create") + @Secured({EDITOR}) @Override public SourceRead createSource(final SourceCreate sourceCreate) { return ApiHelper.execute(() -> sourceHandler.createSource(sourceCreate)); } @Post("/delete") + @Secured({EDITOR}) @Override public void deleteSource(final SourceIdRequestBody sourceIdRequestBody) { ApiHelper.execute(() -> { @@ -70,24 +80,28 @@ public void deleteSource(final SourceIdRequestBody sourceIdRequestBody) { } @Post("/discover_schema") + @Secured({EDITOR}) @Override public SourceDiscoverSchemaRead discoverSchemaForSource(final SourceDiscoverSchemaRequestBody sourceDiscoverSchemaRequestBody) { return ApiHelper.execute(() -> schedulerHandler.discoverSchemaForSourceFromSourceId(sourceDiscoverSchemaRequestBody)); } @Post("/get") + @Secured({READER}) @Override public SourceRead getSource(final SourceIdRequestBody sourceIdRequestBody) { return ApiHelper.execute(() -> sourceHandler.getSource(sourceIdRequestBody)); } @Post("/most_recent_source_actor_catalog") + @Secured({READER}) @Override public ActorCatalogWithUpdatedAt getMostRecentSourceActorCatalog(final SourceIdRequestBody sourceIdRequestBody) { return ApiHelper.execute(() -> sourceHandler.getMostRecentSourceActorCatalogWithUpdatedAt(sourceIdRequestBody)); } @Post("/list") + @Secured({READER}) @Override public SourceReadList listSourcesForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> sourceHandler.listSourcesForWorkspace(workspaceIdRequestBody)); @@ -100,6 +114,7 @@ public SourceReadList searchSources(final SourceSearch sourceSearch) { } @Post("/update") + @Secured({EDITOR}) @Override public SourceRead updateSource(final SourceUpdate sourceUpdate) { return ApiHelper.execute(() -> sourceHandler.updateSource(sourceUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java index 71222c7873c2..97a030dc7f70 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java @@ -4,6 +4,11 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.SourceDefinitionApi; import io.airbyte.api.model.generated.CustomSourceDefinitionCreate; import io.airbyte.api.model.generated.PrivateSourceDefinitionRead; @@ -19,11 +24,14 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/source_definitions") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceDefinitionApiController implements SourceDefinitionApi { private final SourceDefinitionsHandler sourceDefinitionsHandler; @@ -33,12 +41,14 @@ public SourceDefinitionApiController(final SourceDefinitionsHandler sourceDefini } @Post("/create_custom") + @Secured({EDITOR}) @Override public SourceDefinitionRead createCustomSourceDefinition(final CustomSourceDefinitionCreate customSourceDefinitionCreate) { return ApiHelper.execute(() -> sourceDefinitionsHandler.createCustomSourceDefinition(customSourceDefinitionCreate)); } @Post("/delete") + @Secured({ADMIN}) @Override public void deleteSourceDefinition(final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody) { ApiHelper.execute(() -> { @@ -48,48 +58,56 @@ public void deleteSourceDefinition(final SourceDefinitionIdRequestBody sourceDef } @Post("/get") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionRead getSourceDefinition(final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody) { return ApiHelper.execute(() -> sourceDefinitionsHandler.getSourceDefinition(sourceDefinitionIdRequestBody)); } @Post("/get_for_workspace") + @Secured({READER}) @Override public SourceDefinitionRead getSourceDefinitionForWorkspace(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> sourceDefinitionsHandler.getSourceDefinitionForWorkspace(sourceDefinitionIdWithWorkspaceId)); } @Post("/grant_definition") + @Secured({ADMIN}) @Override public PrivateSourceDefinitionRead grantSourceDefinitionToWorkspace(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> sourceDefinitionsHandler.grantSourceDefinitionToWorkspace(sourceDefinitionIdWithWorkspaceId)); } @Post("/list_latest") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionReadList listLatestSourceDefinitions() { return ApiHelper.execute(sourceDefinitionsHandler::listLatestSourceDefinitions); } @Post("/list_private") + @Secured({ADMIN}) @Override public PrivateSourceDefinitionReadList listPrivateSourceDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> sourceDefinitionsHandler.listPrivateSourceDefinitions(workspaceIdRequestBody)); } @Post("/list") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionReadList listSourceDefinitions() { return ApiHelper.execute(sourceDefinitionsHandler::listSourceDefinitions); } @Post("/list_for_workspace") + @Secured({READER}) @Override public SourceDefinitionReadList listSourceDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> sourceDefinitionsHandler.listSourceDefinitionsForWorkspace(workspaceIdRequestBody)); } @Post("/revoke_definition") + @Secured({ADMIN}) @Override public void revokeSourceDefinitionFromWorkspace(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { ApiHelper.execute(() -> { @@ -99,6 +117,7 @@ public void revokeSourceDefinitionFromWorkspace(final SourceDefinitionIdWithWork } @Post("/update") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate sourceDefinitionUpdate) { return ApiHelper.execute(() -> sourceDefinitionsHandler.updateSourceDefinition(sourceDefinitionUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java index 590f7340b61b..e87b5c28bddc 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.SourceDefinitionSpecificationApi; import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; @@ -11,10 +13,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/source_definition_specifications") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceDefinitionSpecificationApiController implements SourceDefinitionSpecificationApi { private final SchedulerHandler schedulerHandler; @@ -24,6 +29,7 @@ public SourceDefinitionSpecificationApiController(final SchedulerHandler schedul } @Post("/get") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> schedulerHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java index 2bba632cf638..52cbc82beb97 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; + import io.airbyte.api.generated.SourceOauthApi; import io.airbyte.api.model.generated.CompleteSourceOauthRequest; import io.airbyte.api.model.generated.OAuthConsentRead; @@ -14,11 +17,14 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.util.Map; @Controller("/api/v1/source_oauths") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceOauthApiController implements SourceOauthApi { private final OAuthHandler oAuthHandler; @@ -28,18 +34,21 @@ public SourceOauthApiController(final OAuthHandler oAuthHandler) { } @Post("/complete_oauth") + @Secured({EDITOR}) @Override public Map completeSourceOAuth(@Body final CompleteSourceOauthRequest completeSourceOauthRequest) { return ApiHelper.execute(() -> oAuthHandler.completeSourceOAuth(completeSourceOauthRequest)); } @Post("/get_consent_url") + @Secured({EDITOR}) @Override public OAuthConsentRead getSourceOAuthConsent(@Body final SourceOauthConsentRequest sourceOauthConsentRequest) { return ApiHelper.execute(() -> oAuthHandler.getSourceOAuthConsent(sourceOauthConsentRequest)); } @Post("/oauth_params/create") + @Secured({ADMIN}) @Override public void setInstancewideSourceOauthParams(@Body final SetInstancewideSourceOauthParamsRequestBody setInstancewideSourceOauthParamsRequestBody) { ApiHelper.execute(() -> { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java index b1b62373aaa1..870a499d5999 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.StateApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ConnectionState; @@ -12,10 +15,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/state") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class StateApiController implements StateApi { private final StateHandler stateHandler; @@ -25,12 +31,14 @@ public StateApiController(final StateHandler stateHandler) { } @Post("/create_or_update") + @Secured({ADMIN}) @Override public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { return ApiHelper.execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); } @Post("/get") + @Secured({READER}) @Override public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> stateHandler.getState(connectionIdRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java index ac13e02015c1..af8c5a3ad316 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java @@ -4,6 +4,10 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.WebBackendApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ConnectionStateType; @@ -23,10 +27,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/web_backend") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class WebBackendApiController implements WebBackendApi { private final WebBackendConnectionsHandler webBackendConnectionsHandler; @@ -42,48 +49,56 @@ public WebBackendApiController(final WebBackendConnectionsHandler webBackendConn } @Post("/state/get_type") + @Secured({READER}) @Override public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); } @Post("/check_updates") + @Secured({READER}) @Override public WebBackendCheckUpdatesRead webBackendCheckUpdates() { return ApiHelper.execute(webBackendCheckUpdatesHandler::checkUpdates); } @Post("/connections/create") + @Secured({EDITOR}) @Override public WebBackendConnectionRead webBackendCreateConnection(final WebBackendConnectionCreate webBackendConnectionCreate) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendCreateConnection(webBackendConnectionCreate)); } @Post("/connections/get") + @Secured({READER}) @Override public WebBackendConnectionRead webBackendGetConnection(final WebBackendConnectionRequestBody webBackendConnectionRequestBody) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendGetConnection(webBackendConnectionRequestBody)); } @Post("/workspace/state") + @Secured({READER}) @Override public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(final WebBackendWorkspaceState webBackendWorkspaceState) { return ApiHelper.execute(() -> webBackendConnectionsHandler.getWorkspaceState(webBackendWorkspaceState)); } @Post("/connections/list") + @Secured({READER}) @Override public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WebBackendConnectionListRequestBody webBackendConnectionListRequestBody) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendListConnectionsForWorkspace(webBackendConnectionListRequestBody)); } @Post("/geographies/list") + @Secured({AUTHENTICATED_USER}) @Override public WebBackendGeographiesListResult webBackendListGeographies() { return ApiHelper.execute(webBackendGeographiesHandler::listGeographiesOSS); } @Post("/connections/update") + @Secured({EDITOR}) @Override public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConnectionUpdate webBackendConnectionUpdate) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendUpdateConnection(webBackendConnectionUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java index 14d3fdc90ad8..1942a4d03afb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.OWNER; + import io.airbyte.api.generated.WorkspaceApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.SlugRequestBody; @@ -19,10 +22,14 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/workspaces") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) +@SuppressWarnings("PMD.AvoidDuplicateLiterals") public class WorkspaceApiController implements WorkspaceApi { private final WorkspacesHandler workspacesHandler; @@ -32,12 +39,14 @@ public WorkspaceApiController(final WorkspacesHandler workspacesHandler) { } @Post("/create") + @Secured({AUTHENTICATED_USER}) @Override public WorkspaceRead createWorkspace(@Body final WorkspaceCreate workspaceCreate) { return ApiHelper.execute(() -> workspacesHandler.createWorkspace(workspaceCreate)); } @Post("/delete") + @Secured({OWNER}) @Override public void deleteWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { ApiHelper.execute(() -> { @@ -47,30 +56,35 @@ public void deleteWorkspace(@Body final WorkspaceIdRequestBody workspaceIdReques } @Post("/get") + @Secured({OWNER}) @Override public WorkspaceRead getWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> workspacesHandler.getWorkspace(workspaceIdRequestBody)); } @Post("/get_by_slug") + @Secured({OWNER}) @Override public WorkspaceRead getWorkspaceBySlug(@Body final SlugRequestBody slugRequestBody) { return ApiHelper.execute(() -> workspacesHandler.getWorkspaceBySlug(slugRequestBody)); } @Post("/list") + @Secured({AUTHENTICATED_USER}) @Override public WorkspaceReadList listWorkspaces() { return ApiHelper.execute(workspacesHandler::listWorkspaces); } @Post("/update") + @Secured({OWNER}) @Override public WorkspaceRead updateWorkspace(@Body final WorkspaceUpdate workspaceUpdate) { return ApiHelper.execute(() -> workspacesHandler.updateWorkspace(workspaceUpdate)); } @Post("/tag_feedback_status_as_done") + @Secured({OWNER}) @Override public void updateWorkspaceFeedback(@Body final WorkspaceGiveFeedback workspaceGiveFeedback) { ApiHelper.execute(() -> { @@ -80,12 +94,14 @@ public void updateWorkspaceFeedback(@Body final WorkspaceGiveFeedback workspaceG } @Post("/update_name") + @Secured({OWNER}) @Override public WorkspaceRead updateWorkspaceName(@Body final WorkspaceUpdateName workspaceUpdateName) { return ApiHelper.execute(() -> workspacesHandler.updateWorkspaceName(workspaceUpdateName)); } @Post("/get_by_connection_id") + @Secured({AUTHENTICATED_USER}) @Override public WorkspaceRead getWorkspaceByConnectionId(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> workspacesHandler.getWorkspaceByConnectionId(connectionIdRequestBody)); diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 53e670c2c523..1c58a63ef546 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -2,19 +2,8 @@ micronaut: application: name: airbyte-server security: - intercept-url-map: - - pattern: /** - httpMethod: GET - access: - - isAnonymous() - - pattern: /** - httpMethod: POST - access: - - isAnonymous() - - pattern: /** - httpMethod: HEAD - access: - - isAnonymous() + authentication-provider-strategy: ALL + enabled: ${API_AUTHORIZATION_ENABLED:false} server: port: 8001 cors: @@ -118,6 +107,32 @@ datasources: username: ${DATABASE_USER} password: ${DATABASE_PASSWORD} +endpoints: + beans: + enabled: true + sensitive: false + env: + enabled: true + sensitive: false + health: + enabled: true + sensitive: false + info: + enabled: true + sensitive: true + loggers: + enabled: true + sensitive: true + refresh: + enabled: false + sensitive: true + routes: + enabled: true + sensitive: false + threaddump: + enabled: true + sensitive: true + flyway: enabled: true datasources: @@ -138,3 +153,10 @@ jooq: jobs: jackson-converter-enabled: true sql-dialect: POSTGRES + +logger: + levels: + # Uncomment to help resolve issues with conditional beans + # io.micronaut.context.condition: DEBUG + # Uncomment to help resolve issues with security beans + # io.micronaut.security: DEBUG diff --git a/deps.toml b/deps.toml index f3a642d9bab4..0fa2bd639d69 100644 --- a/deps.toml +++ b/deps.toml @@ -24,6 +24,7 @@ micronaut = "3.8.1" micronaut-test = "3.8.0" platform-testcontainers = "1.17.3" postgresql = "42.3.5" +reactor = "3.5.2" slf4j = "1.7.36" temporal = "1.17.0" @@ -101,6 +102,7 @@ platform-testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref platform-testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "platform-testcontainers" } postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" } quartz-scheduler = { module = "org.quartz-scheduler:quartz", version = "2.3.2" } +reactor-core = { module = "io.projectreactor:reactor-core", version.ref = "reactor" } s3 = { module = "software.amazon.awssdk:s3", version = "2.16.84" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version = "4.7.3" } From ea5574ac437fe4bf34023a1682fdf01c946897c2 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 23 Jan 2023 10:18:34 -0800 Subject: [PATCH 11/11] Update limitations (#21354) --- docs/integrations/destinations/google-sheets.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/integrations/destinations/google-sheets.md b/docs/integrations/destinations/google-sheets.md index ef4c122966ae..7b836e053508 100644 --- a/docs/integrations/destinations/google-sheets.md +++ b/docs/integrations/destinations/google-sheets.md @@ -107,12 +107,12 @@ Please be aware of the [Google Spreadsheet limitations](#limitations) before you ### Google Sheets Limitations -During the upload process and from the data storage perspective there are some limitations that should be considered beforehands: +During the upload process and from the data storage perspective there are some limitations that should be considered beforehand as [determined by Google here](https://support.google.com/drive/answer/37603): -- **Maximum of 5 Million Cells** +- **Maximum of 10 Million Cells** -A Google Sheets document can have a maximum of 5 million cells. These can be in a single worksheet or in multiple sheets. -In case you already have the 5 million limit reached in fewer columns, it will not allow you to add more columns (and vice versa, i.e., if 5 million cells limit is reached with a certain number of rows, it will not allow more rows). +A Google Sheets document can have a maximum of 10 million cells. These can be in a single worksheet or in multiple sheets. +In case you already have the 10 million limit reached in fewer columns, it will not allow you to add more columns (and vice versa, i.e., if 10 million cells limit is reached with a certain number of rows, it will not allow more rows). - **Maximum of 18,278 Columns**