diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 069d1cd46624..8a6f36d5d775 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -95,19 +95,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.9" - - name: Install Pyenv - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate - name: Install CI scripts # all CI python packages have the prefix "ci_" run: | - source venv/bin/activate pip install --quiet -e ./tools/ci_* - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} run: | - source venv/bin/activate ci_credentials ${{ github.event.inputs.connector }} write-to-storage # normalization also runs destination-specific tests, so fetch their creds also if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then @@ -117,7 +110,6 @@ jobs: fi env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Test ${{ github.event.inputs.connector }} id: test env: @@ -132,7 +124,6 @@ jobs: - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} if: always() run: | - source venv/bin/activate ci_credentials ${{ github.event.inputs.connector }} update-secrets # normalization also runs destination-specific tests, so fetch their creds also if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index f5b3e4e22b3b..44db9b6e9a4d 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.21.0 +current_version = 0.22.0 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 2499aef31e9a..ce125c91cac2 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.22.0 +Surface the resolved manifest in the CDK + ## 0.21.0 Add AvailabilityStrategy concept and use check_availability within CheckStream diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py index ca010c7782ec..40fb1b6fac81 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -81,6 +81,10 @@ def __init__(self, source_config: ConnectionDefinition, debug: bool = False, con if unknown_fields: raise InvalidConnectorDefinitionException(f"Found unknown top-level fields: {unknown_fields}") + @property + def resolved_manifest(self) -> Mapping[str, Any]: + return self._new_source_config + @property def connection_checker(self) -> ConnectionChecker: check = self._new_source_config["check"] if self.construct_using_pydantic_models else self._legacy_source_config["check"] diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 780ae48a81d3..ccb663e991ba 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.21.0", + version="0.22.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRole.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRole.java new file mode 100644 index 000000000000..b2ff02702545 --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRole.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This enum describes the standard auth levels for a given resource. It currently is only used for + * 2 resources Workspace and Instance (i.e. the entire instance or deployment of Airbyte). + * + * In the context of a workspace, there is a 1:1 mapping. + * + * In the context of the instance, there are currently only 3 levels. + * + */ +public enum AuthRole { + + OWNER(500, AuthRoleConstants.OWNER), + ADMIN(400, AuthRoleConstants.ADMIN), + EDITOR(300, AuthRoleConstants.EDITOR), + READER(200, AuthRoleConstants.READER), + AUTHENTICATED_USER(100, AuthRoleConstants.AUTHENTICATED_USER), // ONLY USE WITH INSTANCE RESOURCE! + NONE(0, AuthRoleConstants.NONE); + + private final int authority; + private final String label; + + AuthRole(final int authority, final String label) { + this.authority = authority; + this.label = label; + } + + public int getAuthority() { + return authority; + } + + public String getLabel() { + return label; + } + + /** + * Builds the set of roles based on the provided {@link AuthRole} value. + *

+ * The generated set of auth roles contains the provided {@link AuthRole} (if not {@code null}) and + * any other authentication roles with a lesser {@link #getAuthority()} value. + *

+ * + * @param authRole An {@link AuthRole} (may be {@code null}). + * @return The set of {@link AuthRole}s based on the provided {@link AuthRole}. + */ + public static Set buildAuthRolesSet(final AuthRole authRole) { + final Set authRoles = new HashSet<>(); + + if (authRole != null) { + authRoles.add(authRole); + authRoles.addAll(Stream.of(values()) + .filter(role -> !NONE.equals(role)) + .filter(role -> role.getAuthority() < authRole.getAuthority()) + .collect(Collectors.toSet())); + } + + // Sort final set by descending authority order + return authRoles.stream() + .sorted(Comparator.comparingInt(AuthRole::getAuthority)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java new file mode 100644 index 000000000000..6a206ee0e89e --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +/** + * Collection of constants that defines authorization roles. + */ +public final class AuthRoleConstants { + + public static final String ADMIN = "ADMIN"; + public static final String AUTHENTICATED_USER = "AUTHENTICATED_USER"; + public static final String EDITOR = "EDITOR"; + public static final String OWNER = "OWNER"; + public static final String NONE = "NONE"; + public static final String READER = "READER"; + + private AuthRoleConstants() {} + +} diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/auth/AuthRoleTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/auth/AuthRoleTest.java new file mode 100644 index 000000000000..835488fdd84a --- /dev/null +++ b/airbyte-commons/src/test/java/io/airbyte/commons/auth/AuthRoleTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Set; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link AuthRole} enumeration. + */ +class AuthRoleTest { + + @Test + void testBuildingAuthRoleSet() { + final Set ownerResult = AuthRole.buildAuthRolesSet(AuthRole.OWNER); + assertEquals(5, ownerResult.size()); + assertEquals(Set.of(AuthRole.OWNER, AuthRole.ADMIN, AuthRole.EDITOR, AuthRole.READER, AuthRole.AUTHENTICATED_USER), ownerResult); + + final Set adminResult = AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + assertEquals(4, adminResult.size()); + assertEquals(Set.of(AuthRole.ADMIN, AuthRole.EDITOR, AuthRole.READER, AuthRole.AUTHENTICATED_USER), adminResult); + + final Set editorResult = AuthRole.buildAuthRolesSet(AuthRole.EDITOR); + assertEquals(3, editorResult.size()); + assertEquals(Set.of(AuthRole.EDITOR, AuthRole.READER, AuthRole.AUTHENTICATED_USER), editorResult); + + final Set readerResult = AuthRole.buildAuthRolesSet(AuthRole.READER); + assertEquals(2, readerResult.size()); + assertEquals(Set.of(AuthRole.READER, AuthRole.AUTHENTICATED_USER), readerResult); + + final Set authenticatedUserResult = AuthRole.buildAuthRolesSet(AuthRole.AUTHENTICATED_USER); + assertEquals(1, authenticatedUserResult.size()); + assertEquals(Set.of(AuthRole.AUTHENTICATED_USER), authenticatedUserResult); + + final Set noneResult = AuthRole.buildAuthRolesSet(AuthRole.NONE); + assertEquals(1, noneResult.size()); + assertEquals(Set.of(AuthRole.NONE), noneResult); + + final Set nullResult = AuthRole.buildAuthRolesSet(null); + assertEquals(0, nullResult.size()); + } + +} diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index d84582706ba2..b05591295659 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -40,7 +40,7 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 1.2.11 + dockerImageTag: 1.2.12 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg normalizationConfig: @@ -139,7 +139,7 @@ - name: Google Cloud Storage (GCS) destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a dockerRepository: airbyte/destination-gcs - dockerImageTag: 0.2.12 + dockerImageTag: 0.2.13 documentationUrl: https://docs.airbyte.com/integrations/destinations/gcs icon: googlecloudstorage.svg resourceRequirements: @@ -290,7 +290,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.53 + dockerImageTag: 0.3.54 documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift icon: redshift.svg normalizationConfig: @@ -321,7 +321,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.3.18 + dockerImageTag: 0.3.19 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3 icon: s3.svg resourceRequirements: @@ -348,7 +348,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.43 + dockerImageTag: 0.4.44 documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake icon: snowflake.svg normalizationConfig: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index aa768b304061..7f34e408d5b9 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -621,7 +621,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-bigquery:1.2.11" +- dockerImage: "airbyte/destination-bigquery:1.2.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery" connectionSpecification: @@ -2325,7 +2325,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-gcs:0.2.12" +- dockerImage: "airbyte/destination-gcs:0.2.13" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/gcs" connectionSpecification: @@ -5123,7 +5123,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.53" +- dockerImage: "airbyte/destination-redshift:0.3.54" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift" connectionSpecification: @@ -5492,7 +5492,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.3.18" +- dockerImage: "airbyte/destination-s3:0.3.19" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3" connectionSpecification: @@ -6109,7 +6109,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.4.43" +- dockerImage: "airbyte/destination-snowflake:0.4.44" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/snowflake" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 77abe48daffb..c4fef8a57059 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1019,7 +1019,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.4.27 + dockerImageTag: 0.4.28 documentationUrl: https://docs.airbyte.com/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -1338,7 +1338,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.38 + dockerImageTag: 1.0.39 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 0b6f424cc44b..31d6255539aa 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8113,7 +8113,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.4.27" +- dockerImage: "airbyte/source-mssql:0.4.28" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/mssql" connectionSpecification: @@ -8272,6 +8272,18 @@ \ the \"Snapshot\" level, you must enable the snapshot isolation mode on the database." order: 2 + initial_waiting_seconds: + type: "integer" + title: "Initial Waiting Time in Seconds (Advanced)" + description: "The amount of time the connector will wait when it launches\ + \ to determine if there is new data to sync or not. Defaults to\ + \ 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about\ + \ initial waiting time." + default: 300 + min: 120 + max: 1200 + order: 3 tunnel_method: type: "object" title: "SSH Tunnel Method" @@ -11476,7 +11488,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.38" +- dockerImage: "airbyte/source-postgres:1.0.39" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java index d131470e976e..96681b4216f8 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java @@ -22,9 +22,9 @@ protected BlobStorageOperations() { public abstract String getBucketObjectPath(String namespace, String streamName, DateTime writeDatetime, String customFormat); /** - * Create a storage object where to store data in the destination for a @param objectPath + * Ensure that the bucket specified in the config exists */ - public abstract void createBucketObjectIfNotExists(String objectPath) throws Exception; + public abstract void createBucketIfNotExists() throws Exception; /** * Upload the data files into the storage area. diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java index b5f2037e842b..0873cbb87a01 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java @@ -120,7 +120,7 @@ private static void attemptWriteAndDeleteS3Object(final S3StorageOperations stor final var bucketPath = s3Config.getBucketPath(); if (!Strings.isNullOrEmpty(bucketPath)) { - storageOperations.createBucketObjectIfNotExists(bucketPath); + storageOperations.createBucketIfNotExists(); } s3.putObject(s3Bucket, outputTableName, "check-content"); testIAMUserHasListObjectPermission(s3, s3Bucket); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java index 599463435142..ba9f3a19ceab 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java @@ -8,6 +8,7 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.ListObjectsRequest; @@ -15,12 +16,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.string.Strings; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateManager; import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.integrations.util.ConnectorExceptionUtil; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -95,23 +98,15 @@ public String getBucketObjectPath(final String namespace, final String streamNam /** * Create a directory object at the specified location. Creates the bucket if necessary. - * - * @param objectPath The directory to create. Must be a nonempty string. */ @Override - public void createBucketObjectIfNotExists(final String objectPath) { + public void createBucketIfNotExists() { final String bucket = s3Config.getBucketName(); - final String folderPath = objectPath.endsWith("/") ? objectPath : objectPath + "/"; if (!doesBucketExist(bucket)) { LOGGER.info("Bucket {} does not exist; creating...", bucket); s3Client.createBucket(bucket); LOGGER.info("Bucket {} has been created.", bucket); } - if (!s3Client.doesObjectExist(bucket, folderPath)) { - LOGGER.info("Storage Object {}/{} does not exist in bucket; creating...", bucket, objectPath); - s3Client.putObject(bucket, folderPath, ""); - LOGGER.info("Storage Object {}/{} has been created in bucket.", bucket, objectPath); - } } protected boolean doesBucketExist(final String bucket) { @@ -138,7 +133,18 @@ public String uploadRecordsToBucket(final SerializableBuffer recordsData, exceptionsThrown.add(e); } } - throw new RuntimeException(String.format("Exceptions thrown while uploading records into storage: %s", Strings.join(exceptionsThrown, "\n"))); + // Verifying that ALL exceptions are authentication related before assuming this is a configuration + // issue + // reduces risk of misidentifying errors or reporting a transient error. + final boolean areAllExceptionsAuthExceptions = exceptionsThrown.stream().filter(e -> e instanceof AmazonS3Exception) + .map(s3e -> ((AmazonS3Exception) s3e).getStatusCode()) + .filter(ConnectorExceptionUtil.HTTP_AUTHENTICATION_ERROR_CODES::contains) + .count() == exceptionsThrown.size(); + if (areAllExceptionsAuthExceptions) { + throw new ConfigErrorException(exceptionsThrown.get(0).getMessage(), exceptionsThrown.get(0)); + } else { + throw new RuntimeException(String.format("Exceptions thrown while uploading records into storage: %s", Strings.join(exceptionsThrown, "\n"))); + } } /** diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java index 2db68dff49e6..ea336d8fce65 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.util; +import com.google.common.collect.ImmutableList; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.integrations.base.errors.messages.ErrorMessage; @@ -22,6 +23,8 @@ public class ConnectorExceptionUtil { static final String RECOVERY_CONNECTION_ERROR_MESSAGE = "We're having issues syncing from a Postgres replica that is configured as a hot standby server. " + "Please see https://docs.airbyte.com/integrations/sources/postgres/#sync-data-from-postgres-hot-standby-server for options and workarounds"; + + public static final List HTTP_AUTHENTICATION_ERROR_CODES = ImmutableList.of(401, 403); private static final List> configErrorPredicates = List.of(getConfigErrorPredicate(), getConnectionErrorPredicate(), isRecoveryConnectionExceptionPredicate(), isUnknownColumnInFieldListException()); diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 1cfd9ee045d5..e6cd0710659c 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.11 +LABEL io.airbyte.version=1.2.12 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index f79d950d1d7f..5e0eba63c40d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.11 +LABEL io.airbyte.version=1.2.12 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index 51a269725bef..e955d005b871 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -13,11 +13,14 @@ import com.google.cloud.bigquery.LoadJobConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableId; +import com.google.common.collect.ImmutableList; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.GcsStorageOperations; import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.integrations.util.ConnectorExceptionUtil; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.HashSet; import java.util.List; @@ -84,7 +87,15 @@ public String getStagingFullPath(final String datasetId, final String stream) { public void createSchemaIfNotExists(final String datasetId, final String datasetLocation) { if (!existingSchemas.contains(datasetId)) { LOGGER.info("Creating dataset {}", datasetId); - BigQueryUtils.getOrCreateDataset(bigQuery, datasetId, datasetLocation); + try { + BigQueryUtils.getOrCreateDataset(bigQuery, datasetId, datasetLocation); + } catch (BigQueryException e) { + if (ConnectorExceptionUtil.HTTP_AUTHENTICATION_ERROR_CODES.contains(e.getCode())) { + throw new ConfigErrorException(e.getMessage(), e); + } else { + throw e; + } + } existingSchemas.add(datasetId); } } @@ -99,7 +110,7 @@ public void createTmpTableIfNotExists(final TableId tmpTableId, final Schema tab public void createStageIfNotExists(final String datasetId, final String stream) { final String objectPath = getStagingFullPath(datasetId, stream); LOGGER.info("Creating staging path for stream {} (dataset {}): {}", stream, datasetId, objectPath); - gcsStorageOperations.createBucketObjectIfNotExists(objectPath); + gcsStorageOperations.createBucketIfNotExists(); } @Override diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index 065ff49844ce..3f2c05210cf0 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -84,8 +84,10 @@ class BigQueryDestinationTest { protected static final Path CREDENTIALS_WITH_GCS_STAGING_PATH = Path.of("secrets/credentials-gcs-staging.json"); - protected static final Path[] ALL_PATHS = {CREDENTIALS_WITH_GCS_STAGING_PATH, CREDENTIALS_BAD_PROJECT_PATH, CREDENTIALS_NO_DATASET_CREATION_PATH, + + protected static final Path[] ALL_PATHS = {CREDENTIALS_STANDARD_INSERT_PATH, CREDENTIALS_BAD_PROJECT_PATH, CREDENTIALS_NO_DATASET_CREATION_PATH, CREDENTIALS_NO_EDIT_PUBLIC_SCHEMA_ROLE_PATH, CREDENTIALS_NON_BILLABLE_PROJECT_PATH, CREDENTIALS_WITH_GCS_STAGING_PATH}; + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestinationTest.class); private static final String DATASET_NAME_PREFIX = "bq_dest_integration_test"; diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile index c5534e0a9128..f4d34c04bceb 100644 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ b/airbyte-integrations/connectors/destination-gcs/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.12 +LABEL io.airbyte.version=0.2.13 LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 5f06bd32cc8e..882852e88811 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.53 +LABEL io.airbyte.version=0.3.54 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java index 63abde3f6966..657a582d6eeb 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java @@ -77,7 +77,7 @@ public String getStagingPath(UUID connectionId, String namespace, String streamN public void createStageIfNotExists(JdbcDatabase database, String stageName) throws Exception { final String bucketPath = s3Config.getBucketPath(); final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); - s3StorageOperations.createBucketObjectIfNotExists(prefix + stageName); + s3StorageOperations.createBucketIfNotExists(); } @Override diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index e11e5ea30b84..73f794c6b721 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -40,5 +40,5 @@ RUN /bin/bash -c 'set -e && \ echo "unknown arch" ;\ fi' -LABEL io.airbyte.version=0.3.18 +LABEL io.airbyte.version=0.3.19 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 4b8f627b28ea..da6515de1feb 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -20,5 +20,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=0.4.43 +LABEL io.airbyte.version=0.4.44 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java index 9cdc3058be4e..e9c40de09116 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java @@ -82,7 +82,7 @@ public String uploadRecordsToStage(final JdbcDatabase database, @Override public void createStageIfNotExists(final JdbcDatabase database, final String stageName) { - s3StorageOperations.createBucketObjectIfNotExists(stageName); + s3StorageOperations.createBucketIfNotExists(); } @Override diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 3ec6c69f7c27..97644ffc9f9d 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.27 +LABEL io.airbyte.version=0.4.28 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json index bbdae356015a..55ef10232e9c 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json @@ -139,6 +139,15 @@ "enum": ["Snapshot", "Read Committed"], "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", "order": 2 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 3 } } } diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index b0624aaf167e..44663ce0fe9c 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.27 +LABEL io.airbyte.version=0.4.28 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json index 35b192d2c4de..9be47072c150 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json @@ -150,6 +150,15 @@ "enum": ["Snapshot", "Read Committed"], "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", "order": 2 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 3 } } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json index 4745ff1f922e..0b94887ffc1a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json @@ -150,6 +150,15 @@ "enum": ["Snapshot", "Read Committed"], "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", "order": 2 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 3 } } } diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 6e8c951d7a93..75240c9b2918 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.38 +LABEL io.airbyte.version=1.0.39 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 04a84a8f6750..2d826c4e8376 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.38 +LABEL io.airbyte.version=1.0.39 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java index 23140901e3a7..855a5d34ad2f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java @@ -26,7 +26,7 @@ public class PostgresQueryUtils { public static final String TABLE_ESTIMATE_QUERY = """ - SELECT (SELECT COUNT(*) FROM %s) AS %s, + SELECT (select reltuples::int8 as count from pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace where nspname='%s' AND relname='%s') AS %s, pg_relation_size('%s') AS %s; """; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index d2a3aab3609d..06df1a710c47 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -557,13 +557,20 @@ protected void estimateFullRefreshSyncSize(final JdbcDatabase database, final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, getQuoteString()); - final List tableEstimateResult = getFullTableEstimate(database, fullTableName); + final List tableEstimateResult = getFullTableEstimate(database, fullTableName, schemaName, tableName); if (!tableEstimateResult.isEmpty() && tableEstimateResult.get(0).has(ROW_COUNT_RESULT_COL) && tableEstimateResult.get(0).has(TOTAL_BYTES_RESULT_COL)) { final long syncRowCount = tableEstimateResult.get(0).get(ROW_COUNT_RESULT_COL).asLong(); final long syncByteCount = tableEstimateResult.get(0).get(TOTAL_BYTES_RESULT_COL).asLong(); + // The fast count query can return negative or otherwise invalid results for small tables. In this + // case, we can skip emitting an + // estimate trace altogether since the sync will likely complete quickly. + if (syncRowCount <= 0) { + return; + } + // Here, we double the bytes estimate to account for serialization. Perhaps a better way to do this // is to // read a row and Stringify it to better understand the accurate volume of data sent over the wire. @@ -588,20 +595,23 @@ protected void estimateIncrementalSyncSize(final JdbcDatabase database, final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, getQuoteString()); - final List tableEstimateResult = getFullTableEstimate(database, fullTableName); + final List tableEstimateResult = getFullTableEstimate(database, fullTableName, schemaName, tableName); final long tableRowCount = tableEstimateResult.get(0).get(ROW_COUNT_RESULT_COL).asLong(); final long tableByteCount = tableEstimateResult.get(0).get(TOTAL_BYTES_RESULT_COL).asLong(); + // The fast count query can return negative or otherwise invalid results for small tables. In this + // case, we can skip emitting an + // estimate trace altogether since the sync will likely complete quickly. + if (tableRowCount <= 0) { + return; + } + final long syncRowCount; final long syncByteCount; syncRowCount = getIncrementalTableRowCount(database, fullTableName, cursorInfo, cursorFieldType); - if (tableRowCount == 0) { - syncByteCount = 0; - } else { - syncByteCount = (tableByteCount / tableRowCount) * syncRowCount; - } + syncByteCount = (tableByteCount / tableRowCount) * syncRowCount; // Here, we double the bytes estimate to account for serialization. Perhaps a better way to do this // is to @@ -615,10 +625,14 @@ protected void estimateIncrementalSyncSize(final JdbcDatabase database, } } - private List getFullTableEstimate(final JdbcDatabase database, final String fullTableName) throws SQLException { + private List getFullTableEstimate(final JdbcDatabase database, + final String fullTableName, + final String schemaName, + final String tableName) + throws SQLException { // Construct the table estimate query. final String tableEstimateQuery = - String.format(TABLE_ESTIMATE_QUERY, fullTableName, ROW_COUNT_RESULT_COL, fullTableName, TOTAL_BYTES_RESULT_COL); + String.format(TABLE_ESTIMATE_QUERY, schemaName, tableName, ROW_COUNT_RESULT_COL, fullTableName, TOTAL_BYTES_RESULT_COL); LOGGER.debug("table estimate query: {}", tableEstimateQuery); final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.createStatement().executeQuery(tableEstimateQuery), resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java index d50e731d5e8d..c7c112405b44 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; + import io.airbyte.api.generated.AttemptApi; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.SaveStatsRequestBody; @@ -14,10 +16,13 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/attempt/") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class AttemptApiController implements AttemptApi { private final AttemptHandler attemptHandler; @@ -36,6 +41,7 @@ public InternalOperationResult saveStats(final SaveStatsRequestBody requestBody) @Override @Post(uri = "/set_workflow_in_attempt", processes = MediaType.APPLICATION_JSON) + @Secured({ADMIN}) public InternalOperationResult setWorkflowInAttempt(@Body final SetWorkflowInAttemptRequestBody requestBody) { return ApiHelper.execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index f0de78d03d31..f61b89e41447 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.ConnectionApi; import io.airbyte.api.model.generated.ConnectionCreate; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -21,11 +24,14 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/connections") @Context() @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class ConnectionApiController implements ConnectionApi { private final ConnectionsHandler connectionsHandler; @@ -42,24 +48,28 @@ public ConnectionApiController(final ConnectionsHandler connectionsHandler, @Override @Post(uri = "/create") + @Secured({EDITOR}) public ConnectionRead createConnection(@Body final ConnectionCreate connectionCreate) { return ApiHelper.execute(() -> connectionsHandler.createConnection(connectionCreate)); } @Override @Post(uri = "/update") + @Secured({EDITOR}) public ConnectionRead updateConnection(@Body final ConnectionUpdate connectionUpdate) { return ApiHelper.execute(() -> connectionsHandler.updateConnection(connectionUpdate)); } @Override @Post(uri = "/list") + @Secured({READER}) public ConnectionReadList listConnectionsForWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)); } @Override @Post(uri = "/list_all") + @Secured({READER}) public ConnectionReadList listAllConnectionsForWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> connectionsHandler.listAllConnectionsForWorkspace(workspaceIdRequestBody)); } @@ -72,12 +82,14 @@ public ConnectionReadList searchConnections(@Body final ConnectionSearch connect @Override @Post(uri = "/get") + @Secured({READER}) public ConnectionRead getConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> connectionsHandler.getConnection(connectionIdRequestBody.getConnectionId())); } @Override @Post(uri = "/delete") + @Secured({EDITOR}) public void deleteConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { ApiHelper.execute(() -> { operationsHandler.deleteOperationsForConnection(connectionIdRequestBody); @@ -88,12 +100,14 @@ public void deleteConnection(@Body final ConnectionIdRequestBody connectionIdReq @Override @Post(uri = "/sync") + @Secured({EDITOR}) public JobInfoRead syncConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.syncConnection(connectionIdRequestBody)); } @Override @Post(uri = "/reset") + @Secured({EDITOR}) public JobInfoRead resetConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.resetConnection(connectionIdRequestBody)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java index 6c16cec4a596..7a20184b0e4d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.DestinationApi; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.DestinationCloneRequestBody; @@ -20,24 +23,32 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; -import lombok.AllArgsConstructor; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destinations") @Requires(property = "airbyte.deployment-mode", value = "OSS") -@AllArgsConstructor +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationApiController implements DestinationApi { private final DestinationHandler destinationHandler; private final SchedulerHandler schedulerHandler; + public DestinationApiController(final DestinationHandler destinationHandler, final SchedulerHandler schedulerHandler) { + this.destinationHandler = destinationHandler; + this.schedulerHandler = schedulerHandler; + } + @Post(uri = "/check_connection") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToDestination(@Body final DestinationIdRequestBody destinationIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationId(destinationIdRequestBody)); } @Post(uri = "/check_connection_for_update") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToDestinationForUpdate(@Body final DestinationUpdate destinationUpdate) { return ApiHelper.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationIdForUpdate(destinationUpdate)); @@ -50,12 +61,14 @@ public DestinationRead cloneDestination(@Body final DestinationCloneRequestBody } @Post(uri = "/create") + @Secured({EDITOR}) @Override public DestinationRead createDestination(@Body final DestinationCreate destinationCreate) { return ApiHelper.execute(() -> destinationHandler.createDestination(destinationCreate)); } @Post(uri = "/delete") + @Secured({EDITOR}) @Override public void deleteDestination(@Body final DestinationIdRequestBody destinationIdRequestBody) { ApiHelper.execute(() -> { @@ -65,12 +78,14 @@ public void deleteDestination(@Body final DestinationIdRequestBody destinationId } @Post(uri = "/get") + @Secured({READER}) @Override public DestinationRead getDestination(@Body final DestinationIdRequestBody destinationIdRequestBody) { return ApiHelper.execute(() -> destinationHandler.getDestination(destinationIdRequestBody)); } @Post(uri = "/list") + @Secured({READER}) @Override public DestinationReadList listDestinationsForWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> destinationHandler.listDestinationsForWorkspace(workspaceIdRequestBody)); @@ -83,6 +98,7 @@ public DestinationReadList searchDestinations(@Body final DestinationSearch dest } @Post(uri = "/update") + @Secured({EDITOR}) @Override public DestinationRead updateDestination(@Body final DestinationUpdate destinationUpdate) { return ApiHelper.execute(() -> destinationHandler.updateDestination(destinationUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java index 54409d281021..ab559f6b3b09 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java @@ -4,6 +4,11 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.DestinationDefinitionApi; import io.airbyte.api.model.generated.CustomDestinationDefinitionCreate; import io.airbyte.api.model.generated.DestinationDefinitionIdRequestBody; @@ -19,11 +24,14 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destination_definitions") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationDefinitionApiController implements DestinationDefinitionApi { private final DestinationDefinitionsHandler destinationDefinitionsHandler; @@ -33,12 +41,14 @@ public DestinationDefinitionApiController(final DestinationDefinitionsHandler de } @Post(uri = "/create_custom") + @Secured({EDITOR}) @Override public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { return ApiHelper.execute(() -> destinationDefinitionsHandler.createCustomDestinationDefinition(customDestinationDefinitionCreate)); } @Post(uri = "/delete") + @Secured({ADMIN}) @Override public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { ApiHelper.execute(() -> { @@ -48,18 +58,21 @@ public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody } @Post(uri = "/get") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { return ApiHelper.execute(() -> destinationDefinitionsHandler.getDestinationDefinition(destinationDefinitionIdRequestBody)); } @Post(uri = "/get_for_workspace") + @Secured({READER}) @Override public DestinationDefinitionRead getDestinationDefinitionForWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> destinationDefinitionsHandler.getDestinationDefinitionForWorkspace(destinationDefinitionIdWithWorkspaceId)); } @Post(uri = "/grant_definition") + @Secured({ADMIN}) @Override public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { return ApiHelper @@ -67,30 +80,35 @@ public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace(fi } @Post(uri = "/list") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionReadList listDestinationDefinitions() { return ApiHelper.execute(destinationDefinitionsHandler::listDestinationDefinitions); } @Post(uri = "/list_for_workspace") + @Secured({READER}) @Override public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> destinationDefinitionsHandler.listDestinationDefinitionsForWorkspace(workspaceIdRequestBody)); } @Post(uri = "/list_latest") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionReadList listLatestDestinationDefinitions() { return ApiHelper.execute(destinationDefinitionsHandler::listLatestDestinationDefinitions); } @Post(uri = "/list_private") + @Secured({ADMIN}) @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); } @Post(uri = "/revoke_definition") + @Secured({ADMIN}) @Override public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { ApiHelper.execute(() -> { @@ -100,6 +118,7 @@ public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinition } @Post(uri = "/update") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { return ApiHelper.execute(() -> destinationDefinitionsHandler.updateDestinationDefinition(destinationDefinitionUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java index 54492c4a905f..d93bc6dcbe99 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.DestinationDefinitionSpecificationApi; import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; @@ -11,10 +13,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destination_definition_specifications") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationDefinitionSpecificationApiController implements DestinationDefinitionSpecificationApi { private final SchedulerHandler schedulerHandler; @@ -24,6 +29,7 @@ public DestinationDefinitionSpecificationApiController(final SchedulerHandler sc } @Post("/get") + @Secured({AUTHENTICATED_USER}) @Override public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java index 33318e14efcb..58286dfacd48 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; + import io.airbyte.api.generated.DestinationOauthApi; import io.airbyte.api.model.generated.CompleteDestinationOAuthRequest; import io.airbyte.api.model.generated.DestinationOauthConsentRequest; @@ -14,12 +17,15 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.util.Map; @Controller("/api/v1/destination_oauths") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationOauthApiController implements DestinationOauthApi { private final OAuthHandler oAuthHandler; @@ -29,18 +35,21 @@ public DestinationOauthApiController(final OAuthHandler oAuthHandler) { } @Post("/complete_oauth") + @Secured({EDITOR}) @Override public Map completeDestinationOAuth(final CompleteDestinationOAuthRequest completeDestinationOAuthRequest) { return ApiHelper.execute(() -> oAuthHandler.completeDestinationOAuth(completeDestinationOAuthRequest)); } @Post("/get_consent_url") + @Secured({EDITOR}) @Override public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsentRequest destinationOauthConsentRequest) { return ApiHelper.execute(() -> oAuthHandler.getDestinationOAuthConsent(destinationOauthConsentRequest)); } @Post("/oauth_params/create") + @Secured({ADMIN}) @Override public void setInstancewideDestinationOauthParams(final SetInstancewideDestinationOauthParamsRequestBody setInstancewideDestinationOauthParamsRequestBody) { ApiHelper.execute(() -> { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java index a5b46d39742b..3ffb6851cbc5 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java @@ -11,10 +11,13 @@ import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/health") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_ANONYMOUS) public class HealthApiController implements HealthApi { private final HealthCheckHandler healthCheckHandler; diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java index 9fadce045d05..b930472e5d23 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java @@ -4,6 +4,10 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.JobsApi; import io.airbyte.api.model.generated.AttemptNormalizationStatusReadList; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -20,11 +24,14 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/jobs") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class JobsApiController implements JobsApi { private final JobHistoryHandler jobHistoryHandler; @@ -36,42 +43,49 @@ public JobsApiController(final JobHistoryHandler jobHistoryHandler, final Schedu } @Post("/cancel") + @Secured({EDITOR}) @Override public JobInfoRead cancelJob(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.cancelJob(jobIdRequestBody)); } @Post("/get_normalization_status") + @Secured({ADMIN}) @Override public AttemptNormalizationStatusReadList getAttemptNormalizationStatusesForJob(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getAttemptNormalizationStatuses(jobIdRequestBody)); } @Post("/get_debug_info") + @Secured({READER}) @Override public JobDebugInfoRead getJobDebugInfo(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getJobDebugInfo(jobIdRequestBody)); } @Post("/get") + @Secured({READER}) @Override public JobInfoRead getJobInfo(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getJobInfo(jobIdRequestBody)); } @Post("/get_light") + @Secured({READER}) @Override public JobInfoLightRead getJobInfoLight(final JobIdRequestBody jobIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getJobInfoLight(jobIdRequestBody)); } @Post("/get_last_replication_job") + @Secured({READER}) @Override public JobOptionalRead getLastReplicationJob(final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.getLastReplicationJob(connectionIdRequestBody)); } @Post("/list") + @Secured({READER}) @Override public JobReadList listJobsFor(final JobListRequestBody jobListRequestBody) { return ApiHelper.execute(() -> jobHistoryHandler.listJobsFor(jobListRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java index 126ed8202693..13257412cf04 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; + import io.airbyte.api.generated.LogsApi; import io.airbyte.api.model.generated.LogsRequestBody; import io.airbyte.server.handlers.LogsHandler; @@ -11,12 +13,15 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.io.File; @Controller("/api/v1/logs") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class LogsApiController implements LogsApi { private final LogsHandler logsHandler; @@ -26,6 +31,7 @@ public LogsApiController(final LogsHandler logsHandler) { } @Post("/get") + @Secured({ADMIN}) @Override public File getLogs(final LogsRequestBody logsRequestBody) { return ApiHelper.execute(() -> logsHandler.getLogs(logsRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java index a59dea440b05..0a089a21b762 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java @@ -11,6 +11,8 @@ import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Error; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; /** * Custom controller that handles global 404 responses for unknown/unmapped paths. @@ -18,6 +20,7 @@ @Controller("/api/notfound") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_ANONYMOUS) public class NotFoundController { @Error(status = HttpStatus.NOT_FOUND, diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java index 6af6a1cdf242..c81799b9ac92 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.NotificationsApi; import io.airbyte.api.model.generated.Notification; import io.airbyte.api.model.generated.NotificationRead; @@ -12,10 +14,13 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/notifications/try") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class NotificationsApiController implements NotificationsApi { private final WorkspacesHandler workspacesHandler; @@ -25,6 +30,7 @@ public NotificationsApiController(final WorkspacesHandler workspacesHandler) { } @Post + @Secured({AUTHENTICATED_USER}) @Override public NotificationRead tryNotificationConfig(@Body final Notification notification) { return ApiHelper.execute(() -> workspacesHandler.tryNotification(notification)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java index 3aa984d7b2c7..5e574a23deb9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java @@ -4,16 +4,21 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.OpenapiApi; import io.airbyte.server.handlers.OpenApiConfigHandler; import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.io.File; @Controller("/api/v1/openapi") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class OpenapiApiController implements OpenapiApi { private final OpenApiConfigHandler openApiConfigHandler; @@ -23,6 +28,7 @@ public OpenapiApiController(final OpenApiConfigHandler openApiConfigHandler) { } @Get(produces = "text/plain") + @Secured({AUTHENTICATED_USER}) @Override public File getOpenApiSpec() { return ApiHelper.execute(openApiConfigHandler::getFile); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java index a892ca03d47b..172fbb76e40a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java @@ -4,6 +4,10 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.OperationApi; import io.airbyte.api.model.generated.CheckOperationRead; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -18,10 +22,13 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/operations") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class OperationApiController implements OperationApi { private final OperationsHandler operationsHandler; @@ -31,6 +38,7 @@ public OperationApiController(final OperationsHandler operationsHandler) { } @Post("/check") + @Secured({AUTHENTICATED_USER}) @Override public CheckOperationRead checkOperation(@Body final OperatorConfiguration operatorConfiguration) { return ApiHelper.execute(() -> operationsHandler.checkOperation(operatorConfiguration)); @@ -38,11 +46,13 @@ public CheckOperationRead checkOperation(@Body final OperatorConfiguration opera @Post("/create") @Override + @Secured({EDITOR}) public OperationRead createOperation(@Body final OperationCreate operationCreate) { return ApiHelper.execute(() -> operationsHandler.createOperation(operationCreate)); } @Post("/delete") + @Secured({EDITOR}) @Override public void deleteOperation(@Body final OperationIdRequestBody operationIdRequestBody) { ApiHelper.execute(() -> { @@ -52,18 +62,21 @@ public void deleteOperation(@Body final OperationIdRequestBody operationIdReques } @Post("/get") + @Secured({READER}) @Override public OperationRead getOperation(@Body final OperationIdRequestBody operationIdRequestBody) { return ApiHelper.execute(() -> operationsHandler.getOperation(operationIdRequestBody)); } @Post("/list") + @Secured({READER}) @Override public OperationReadList listOperationsForConnection(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> operationsHandler.listOperationsForConnection(connectionIdRequestBody)); } @Post("/update") + @Secured({EDITOR}) @Override public OperationRead updateOperation(@Body final OperationUpdate operationUpdate) { return ApiHelper.execute(() -> operationsHandler.updateOperation(operationUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java index 53019774aeca..365dc04afc46 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; + import io.airbyte.api.generated.SchedulerApi; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.DestinationCoreConfig; @@ -13,10 +16,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/scheduler") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SchedulerApiController implements SchedulerApi { private final SchedulerHandler schedulerHandler; @@ -26,18 +32,21 @@ public SchedulerApiController(final SchedulerHandler schedulerHandler) { } @Post("/destinations/check_connection") + @Secured({AUTHENTICATED_USER}) @Override public CheckConnectionRead executeDestinationCheckConnection(final DestinationCoreConfig destinationCoreConfig) { return ApiHelper.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationCreate(destinationCoreConfig)); } @Post("/sources/check_connection") + @Secured({AUTHENTICATED_USER}) @Override public CheckConnectionRead executeSourceCheckConnection(final SourceCoreConfig sourceCoreConfig) { return ApiHelper.execute(() -> schedulerHandler.checkSourceConnectionFromSourceCreate(sourceCoreConfig)); } @Post("/sources/discover_schema") + @Secured({EDITOR}) @Override public SourceDiscoverSchemaRead executeSourceDiscoverSchema(final SourceCoreConfig sourceCoreConfig) { return ApiHelper.execute(() -> schedulerHandler.discoverSchemaForSourceFromSourceCreate(sourceCoreConfig)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java index 64cf7c58dfd6..0c680f742989 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.SourceApi; import io.airbyte.api.model.generated.ActorCatalogWithUpdatedAt; import io.airbyte.api.model.generated.CheckConnectionRead; @@ -22,10 +25,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/sources") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceApiController implements SourceApi { private final SchedulerHandler schedulerHandler; @@ -37,12 +43,14 @@ public SourceApiController(final SchedulerHandler schedulerHandler, final Source } @Post("/check_connection") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToSource(final SourceIdRequestBody sourceIdRequestBody) { return ApiHelper.execute(() -> schedulerHandler.checkSourceConnectionFromSourceId(sourceIdRequestBody)); } @Post("/check_connection_for_update") + @Secured({EDITOR}) @Override public CheckConnectionRead checkConnectionToSourceForUpdate(final SourceUpdate sourceUpdate) { return ApiHelper.execute(() -> schedulerHandler.checkSourceConnectionFromSourceIdForUpdate(sourceUpdate)); @@ -55,12 +63,14 @@ public SourceRead cloneSource(final SourceCloneRequestBody sourceCloneRequestBod } @Post("/create") + @Secured({EDITOR}) @Override public SourceRead createSource(final SourceCreate sourceCreate) { return ApiHelper.execute(() -> sourceHandler.createSource(sourceCreate)); } @Post("/delete") + @Secured({EDITOR}) @Override public void deleteSource(final SourceIdRequestBody sourceIdRequestBody) { ApiHelper.execute(() -> { @@ -70,24 +80,28 @@ public void deleteSource(final SourceIdRequestBody sourceIdRequestBody) { } @Post("/discover_schema") + @Secured({EDITOR}) @Override public SourceDiscoverSchemaRead discoverSchemaForSource(final SourceDiscoverSchemaRequestBody sourceDiscoverSchemaRequestBody) { return ApiHelper.execute(() -> schedulerHandler.discoverSchemaForSourceFromSourceId(sourceDiscoverSchemaRequestBody)); } @Post("/get") + @Secured({READER}) @Override public SourceRead getSource(final SourceIdRequestBody sourceIdRequestBody) { return ApiHelper.execute(() -> sourceHandler.getSource(sourceIdRequestBody)); } @Post("/most_recent_source_actor_catalog") + @Secured({READER}) @Override public ActorCatalogWithUpdatedAt getMostRecentSourceActorCatalog(final SourceIdRequestBody sourceIdRequestBody) { return ApiHelper.execute(() -> sourceHandler.getMostRecentSourceActorCatalogWithUpdatedAt(sourceIdRequestBody)); } @Post("/list") + @Secured({READER}) @Override public SourceReadList listSourcesForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> sourceHandler.listSourcesForWorkspace(workspaceIdRequestBody)); @@ -100,6 +114,7 @@ public SourceReadList searchSources(final SourceSearch sourceSearch) { } @Post("/update") + @Secured({EDITOR}) @Override public SourceRead updateSource(final SourceUpdate sourceUpdate) { return ApiHelper.execute(() -> sourceHandler.updateSource(sourceUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java index 71222c7873c2..97a030dc7f70 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java @@ -4,6 +4,11 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.SourceDefinitionApi; import io.airbyte.api.model.generated.CustomSourceDefinitionCreate; import io.airbyte.api.model.generated.PrivateSourceDefinitionRead; @@ -19,11 +24,14 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/source_definitions") @Requires(property = "airbyte.deployment-mode", value = "OSS") @Context +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceDefinitionApiController implements SourceDefinitionApi { private final SourceDefinitionsHandler sourceDefinitionsHandler; @@ -33,12 +41,14 @@ public SourceDefinitionApiController(final SourceDefinitionsHandler sourceDefini } @Post("/create_custom") + @Secured({EDITOR}) @Override public SourceDefinitionRead createCustomSourceDefinition(final CustomSourceDefinitionCreate customSourceDefinitionCreate) { return ApiHelper.execute(() -> sourceDefinitionsHandler.createCustomSourceDefinition(customSourceDefinitionCreate)); } @Post("/delete") + @Secured({ADMIN}) @Override public void deleteSourceDefinition(final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody) { ApiHelper.execute(() -> { @@ -48,48 +58,56 @@ public void deleteSourceDefinition(final SourceDefinitionIdRequestBody sourceDef } @Post("/get") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionRead getSourceDefinition(final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody) { return ApiHelper.execute(() -> sourceDefinitionsHandler.getSourceDefinition(sourceDefinitionIdRequestBody)); } @Post("/get_for_workspace") + @Secured({READER}) @Override public SourceDefinitionRead getSourceDefinitionForWorkspace(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> sourceDefinitionsHandler.getSourceDefinitionForWorkspace(sourceDefinitionIdWithWorkspaceId)); } @Post("/grant_definition") + @Secured({ADMIN}) @Override public PrivateSourceDefinitionRead grantSourceDefinitionToWorkspace(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> sourceDefinitionsHandler.grantSourceDefinitionToWorkspace(sourceDefinitionIdWithWorkspaceId)); } @Post("/list_latest") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionReadList listLatestSourceDefinitions() { return ApiHelper.execute(sourceDefinitionsHandler::listLatestSourceDefinitions); } @Post("/list_private") + @Secured({ADMIN}) @Override public PrivateSourceDefinitionReadList listPrivateSourceDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> sourceDefinitionsHandler.listPrivateSourceDefinitions(workspaceIdRequestBody)); } @Post("/list") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionReadList listSourceDefinitions() { return ApiHelper.execute(sourceDefinitionsHandler::listSourceDefinitions); } @Post("/list_for_workspace") + @Secured({READER}) @Override public SourceDefinitionReadList listSourceDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> sourceDefinitionsHandler.listSourceDefinitionsForWorkspace(workspaceIdRequestBody)); } @Post("/revoke_definition") + @Secured({ADMIN}) @Override public void revokeSourceDefinitionFromWorkspace(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { ApiHelper.execute(() -> { @@ -99,6 +117,7 @@ public void revokeSourceDefinitionFromWorkspace(final SourceDefinitionIdWithWork } @Post("/update") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate sourceDefinitionUpdate) { return ApiHelper.execute(() -> sourceDefinitionsHandler.updateSourceDefinition(sourceDefinitionUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java index 590f7340b61b..e87b5c28bddc 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java @@ -4,6 +4,8 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; + import io.airbyte.api.generated.SourceDefinitionSpecificationApi; import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; @@ -11,10 +13,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/source_definition_specifications") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceDefinitionSpecificationApiController implements SourceDefinitionSpecificationApi { private final SchedulerHandler schedulerHandler; @@ -24,6 +29,7 @@ public SourceDefinitionSpecificationApiController(final SchedulerHandler schedul } @Post("/get") + @Secured({AUTHENTICATED_USER}) @Override public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { return ApiHelper.execute(() -> schedulerHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java index 2bba632cf638..52cbc82beb97 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; + import io.airbyte.api.generated.SourceOauthApi; import io.airbyte.api.model.generated.CompleteSourceOauthRequest; import io.airbyte.api.model.generated.OAuthConsentRead; @@ -14,11 +17,14 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import java.util.Map; @Controller("/api/v1/source_oauths") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class SourceOauthApiController implements SourceOauthApi { private final OAuthHandler oAuthHandler; @@ -28,18 +34,21 @@ public SourceOauthApiController(final OAuthHandler oAuthHandler) { } @Post("/complete_oauth") + @Secured({EDITOR}) @Override public Map completeSourceOAuth(@Body final CompleteSourceOauthRequest completeSourceOauthRequest) { return ApiHelper.execute(() -> oAuthHandler.completeSourceOAuth(completeSourceOauthRequest)); } @Post("/get_consent_url") + @Secured({EDITOR}) @Override public OAuthConsentRead getSourceOAuthConsent(@Body final SourceOauthConsentRequest sourceOauthConsentRequest) { return ApiHelper.execute(() -> oAuthHandler.getSourceOAuthConsent(sourceOauthConsentRequest)); } @Post("/oauth_params/create") + @Secured({ADMIN}) @Override public void setInstancewideSourceOauthParams(@Body final SetInstancewideSourceOauthParamsRequestBody setInstancewideSourceOauthParamsRequestBody) { ApiHelper.execute(() -> { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java index b1b62373aaa1..870a499d5999 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.ADMIN; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.StateApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ConnectionState; @@ -12,10 +15,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/state") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class StateApiController implements StateApi { private final StateHandler stateHandler; @@ -25,12 +31,14 @@ public StateApiController(final StateHandler stateHandler) { } @Post("/create_or_update") + @Secured({ADMIN}) @Override public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { return ApiHelper.execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); } @Post("/get") + @Secured({READER}) @Override public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> stateHandler.getState(connectionIdRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java index ac13e02015c1..af8c5a3ad316 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java @@ -4,6 +4,10 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; + import io.airbyte.api.generated.WebBackendApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ConnectionStateType; @@ -23,10 +27,13 @@ import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/web_backend") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) public class WebBackendApiController implements WebBackendApi { private final WebBackendConnectionsHandler webBackendConnectionsHandler; @@ -42,48 +49,56 @@ public WebBackendApiController(final WebBackendConnectionsHandler webBackendConn } @Post("/state/get_type") + @Secured({READER}) @Override public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); } @Post("/check_updates") + @Secured({READER}) @Override public WebBackendCheckUpdatesRead webBackendCheckUpdates() { return ApiHelper.execute(webBackendCheckUpdatesHandler::checkUpdates); } @Post("/connections/create") + @Secured({EDITOR}) @Override public WebBackendConnectionRead webBackendCreateConnection(final WebBackendConnectionCreate webBackendConnectionCreate) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendCreateConnection(webBackendConnectionCreate)); } @Post("/connections/get") + @Secured({READER}) @Override public WebBackendConnectionRead webBackendGetConnection(final WebBackendConnectionRequestBody webBackendConnectionRequestBody) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendGetConnection(webBackendConnectionRequestBody)); } @Post("/workspace/state") + @Secured({READER}) @Override public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(final WebBackendWorkspaceState webBackendWorkspaceState) { return ApiHelper.execute(() -> webBackendConnectionsHandler.getWorkspaceState(webBackendWorkspaceState)); } @Post("/connections/list") + @Secured({READER}) @Override public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WebBackendConnectionListRequestBody webBackendConnectionListRequestBody) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendListConnectionsForWorkspace(webBackendConnectionListRequestBody)); } @Post("/geographies/list") + @Secured({AUTHENTICATED_USER}) @Override public WebBackendGeographiesListResult webBackendListGeographies() { return ApiHelper.execute(webBackendGeographiesHandler::listGeographiesOSS); } @Post("/connections/update") + @Secured({EDITOR}) @Override public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConnectionUpdate webBackendConnectionUpdate) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendUpdateConnection(webBackendConnectionUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java index 14d3fdc90ad8..1942a4d03afb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java @@ -4,6 +4,9 @@ package io.airbyte.server.apis; +import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; +import static io.airbyte.commons.auth.AuthRoleConstants.OWNER; + import io.airbyte.api.generated.WorkspaceApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.SlugRequestBody; @@ -19,10 +22,14 @@ import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/workspaces") @Requires(property = "airbyte.deployment-mode", value = "OSS") +@Secured(SecurityRule.IS_AUTHENTICATED) +@SuppressWarnings("PMD.AvoidDuplicateLiterals") public class WorkspaceApiController implements WorkspaceApi { private final WorkspacesHandler workspacesHandler; @@ -32,12 +39,14 @@ public WorkspaceApiController(final WorkspacesHandler workspacesHandler) { } @Post("/create") + @Secured({AUTHENTICATED_USER}) @Override public WorkspaceRead createWorkspace(@Body final WorkspaceCreate workspaceCreate) { return ApiHelper.execute(() -> workspacesHandler.createWorkspace(workspaceCreate)); } @Post("/delete") + @Secured({OWNER}) @Override public void deleteWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { ApiHelper.execute(() -> { @@ -47,30 +56,35 @@ public void deleteWorkspace(@Body final WorkspaceIdRequestBody workspaceIdReques } @Post("/get") + @Secured({OWNER}) @Override public WorkspaceRead getWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { return ApiHelper.execute(() -> workspacesHandler.getWorkspace(workspaceIdRequestBody)); } @Post("/get_by_slug") + @Secured({OWNER}) @Override public WorkspaceRead getWorkspaceBySlug(@Body final SlugRequestBody slugRequestBody) { return ApiHelper.execute(() -> workspacesHandler.getWorkspaceBySlug(slugRequestBody)); } @Post("/list") + @Secured({AUTHENTICATED_USER}) @Override public WorkspaceReadList listWorkspaces() { return ApiHelper.execute(workspacesHandler::listWorkspaces); } @Post("/update") + @Secured({OWNER}) @Override public WorkspaceRead updateWorkspace(@Body final WorkspaceUpdate workspaceUpdate) { return ApiHelper.execute(() -> workspacesHandler.updateWorkspace(workspaceUpdate)); } @Post("/tag_feedback_status_as_done") + @Secured({OWNER}) @Override public void updateWorkspaceFeedback(@Body final WorkspaceGiveFeedback workspaceGiveFeedback) { ApiHelper.execute(() -> { @@ -80,12 +94,14 @@ public void updateWorkspaceFeedback(@Body final WorkspaceGiveFeedback workspaceG } @Post("/update_name") + @Secured({OWNER}) @Override public WorkspaceRead updateWorkspaceName(@Body final WorkspaceUpdateName workspaceUpdateName) { return ApiHelper.execute(() -> workspacesHandler.updateWorkspaceName(workspaceUpdateName)); } @Post("/get_by_connection_id") + @Secured({AUTHENTICATED_USER}) @Override public WorkspaceRead getWorkspaceByConnectionId(@Body final ConnectionIdRequestBody connectionIdRequestBody) { return ApiHelper.execute(() -> workspacesHandler.getWorkspaceByConnectionId(connectionIdRequestBody)); diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 53e670c2c523..1c58a63ef546 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -2,19 +2,8 @@ micronaut: application: name: airbyte-server security: - intercept-url-map: - - pattern: /** - httpMethod: GET - access: - - isAnonymous() - - pattern: /** - httpMethod: POST - access: - - isAnonymous() - - pattern: /** - httpMethod: HEAD - access: - - isAnonymous() + authentication-provider-strategy: ALL + enabled: ${API_AUTHORIZATION_ENABLED:false} server: port: 8001 cors: @@ -118,6 +107,32 @@ datasources: username: ${DATABASE_USER} password: ${DATABASE_PASSWORD} +endpoints: + beans: + enabled: true + sensitive: false + env: + enabled: true + sensitive: false + health: + enabled: true + sensitive: false + info: + enabled: true + sensitive: true + loggers: + enabled: true + sensitive: true + refresh: + enabled: false + sensitive: true + routes: + enabled: true + sensitive: false + threaddump: + enabled: true + sensitive: true + flyway: enabled: true datasources: @@ -138,3 +153,10 @@ jooq: jobs: jackson-converter-enabled: true sql-dialect: POSTGRES + +logger: + levels: + # Uncomment to help resolve issues with conditional beans + # io.micronaut.context.condition: DEBUG + # Uncomment to help resolve issues with security beans + # io.micronaut.security: DEBUG diff --git a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss index a3166da8dda7..524972b7ce19 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss +++ b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.module.scss @@ -2,7 +2,11 @@ .container { margin: 13px auto 0; - padding-bottom: variables.$spacing-page-bottom; + padding-bottom: variables.$spacing-xl; + + &.cloud { + padding-bottom: variables.$spacing-page-bottom-cloud; + } &:not(.big) { width: 80%; diff --git a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx index 64af7f776f97..1e1c4a051ec5 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx +++ b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx @@ -1,6 +1,8 @@ import classNames from "classnames"; import { PropsWithChildren } from "react"; +import { isCloudApp } from "utils/app"; + import styles from "./FormPageContent.module.scss"; interface FormPageContentProps { @@ -8,7 +10,14 @@ interface FormPageContentProps { } const FormPageContent: React.FC> = ({ big, children }) => ( -
{children}
+
+ {children} +
); export default FormPageContent; diff --git a/airbyte-webapp/src/components/base/Titles/Titles.tsx b/airbyte-webapp/src/components/base/Titles/Titles.tsx index 9d2a5ee0fd9d..50630f7ce478 100644 --- a/airbyte-webapp/src/components/base/Titles/Titles.tsx +++ b/airbyte-webapp/src/components/base/Titles/Titles.tsx @@ -19,12 +19,6 @@ const H1 = styled.h1` margin: 0; `; -/** @deprecated Use `` */ -export const H3 = styled(H1).attrs({ as: "h3" })` - font-size: 20px; - line-height: 24px; -`; - /** @deprecated Use `` */ export const H5 = styled(H1).attrs({ as: "h5" })` font-size: ${({ theme }) => theme.h5?.fontSize || "16px"}; diff --git a/airbyte-webapp/src/components/base/Titles/index.tsx b/airbyte-webapp/src/components/base/Titles/index.tsx index 502ba2ea7036..c6f9a9079c5f 100644 --- a/airbyte-webapp/src/components/base/Titles/index.tsx +++ b/airbyte-webapp/src/components/base/Titles/index.tsx @@ -1 +1 @@ -export { H3, H5 } from "./Titles"; +export { H5 } from "./Titles"; diff --git a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss index 22c55a7f45fb..185526591e75 100644 --- a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss +++ b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.module.scss @@ -9,6 +9,7 @@ } .contentContainer { + flex: 1; max-width: 100%; overflow-x: auto; padding-top: variables.$spacing-lg; @@ -17,6 +18,10 @@ .content { overflow-y: auto; height: 100%; - padding: 0 variables.$spacing-xl variables.$spacing-page-bottom; + padding: 0 variables.$spacing-xl variables.$spacing-xl; min-width: variables.$main-page-content-min-width; + + &.cloud { + padding-bottom: variables.$spacing-page-bottom-cloud; + } } diff --git a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx index 047e1e14dbe1..71091c50b3cb 100644 --- a/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx +++ b/airbyte-webapp/src/components/common/MainPageWithScroll/MainPageWithScroll.tsx @@ -1,5 +1,8 @@ +import classNames from "classnames"; import React from "react"; +import { isCloudApp } from "utils/app"; + import styles from "./MainPageWithScroll.module.scss"; /** @@ -20,7 +23,7 @@ export const MainPageWithScroll: React.FC = ({ headTitl {pageTitle}
-
{children}
+
{children}
); diff --git a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss index ce45b41bb4f7..343ffae519be 100644 --- a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss +++ b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.module.scss @@ -1,5 +1,12 @@ +@use "scss/colors"; @use "scss/variables"; .formCard { - padding: 22px 27px variables.$spacing-xl 24px; + padding: variables.$spacing-xl; +} + +.editControls { + border-top: variables.$border-thin solid colors.$grey-100; + margin: 0 -#{variables.$spacing-xl}; + padding: variables.$spacing-xl variables.$spacing-xl 0; } diff --git a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx index a0be5283d333..3533cafa8a4a 100644 --- a/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionEditFormCard/ConnectionEditFormCard.tsx @@ -13,7 +13,6 @@ import EditControls from "../ConnectionForm/EditControls"; import styles from "./ConnectionEditFormCard.module.scss"; interface FormCardProps extends CollapsibleCardProps { - bottomSeparator?: boolean; form: FormikConfig; submitDisabled?: boolean; } @@ -21,7 +20,6 @@ interface FormCardProps extends CollapsibleCardProps { export const ConnectionEditFormCard = ({ children, form, - bottomSeparator = true, submitDisabled, ...props }: React.PropsWithChildren>) => { @@ -45,10 +43,9 @@ export const ConnectionEditFormCard = ({
{children} -
+
{mode !== "readonly" && ( theme.darkPrimaryColor}; - justify-content: center; -`; - -const Loader = styled.div` - margin-right: 10px; -`; - const Success = styled(StatusIcon)` width: 26px; min-width: 26px; @@ -61,19 +44,8 @@ const ErrorText = styled.div` `; const CreateControls: React.FC = ({ isSubmitting, errorMessage, isValid }) => { - if (isSubmitting) { - return ( - - - - - - - ); - } - return ( - + {errorMessage ? ( @@ -86,11 +58,11 @@ const CreateControls: React.FC = ({ isSubmitting, errorMess
)}
-
- + ); }; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss index 2afd4f0946a3..1610ff1d8d2c 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss +++ b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss @@ -1,29 +1,5 @@ -@use "scss/colors"; @use "scss/variables"; -.content { - display: flex; - justify-content: flex-end; - align-items: center; - flex-direction: row; - margin-top: variables.$spacing-lg; - gap: variables.$spacing-lg; - padding: variables.$spacing-sm; -} - -.controlButton { - margin-left: variables.$spacing-md; -} - -// currently only implemented on transformation view form card, margins are specific to that implementation -// todo: standardize the margin sizes here -.line { - min-width: 100%; - height: variables.$border-thin; - background: colors.$grey-100; - margin: variables.$spacing-lg -27px 0 -24px; -} - -.buttonsContainer { - display: flex; +.container { + margin-top: variables.$spacing-md; } diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx index fc78785d6458..6fa6fe33cd1b 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx @@ -2,6 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; import styles from "./EditControls.module.scss"; import { ResponseMessage } from "./ResponseMessage"; @@ -14,7 +15,6 @@ interface EditControlProps { successMessage?: React.ReactNode; errorMessage?: React.ReactNode; enableControls?: boolean; - withLine?: boolean; } const EditControls: React.FC = ({ @@ -25,33 +25,28 @@ const EditControls: React.FC = ({ successMessage, errorMessage, enableControls, - withLine, }) => { return ( - <> - {withLine &&
} -
- -
- - -
-
- + + + + + + + ); }; diff --git a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.module.scss b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.module.scss deleted file mode 100644 index 87ff85fde155..000000000000 --- a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.collapsibleCard { - margin-bottom: 10px; -} diff --git a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx index 4b0dfdaeec9d..52b12607e2bd 100644 --- a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx +++ b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx @@ -1,14 +1,11 @@ import { faChevronRight } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import classNames from "classnames"; import React from "react"; import { useToggle } from "react-use"; import styled from "styled-components"; import { Card } from "components/ui/Card"; -import styles from "./CollapsibleCard.module.scss"; - const CardHeader = styled.div` display: flex; justify-content: space-between; @@ -40,7 +37,6 @@ export const CollapsibleCard: React.FC = ({ return ( {title} diff --git a/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts b/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts index 32c11bf9f1c6..df0f42e91754 100644 --- a/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts +++ b/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts @@ -20,6 +20,8 @@ import { signInWithPopup, GoogleAuthProvider, GithubAuthProvider, + getIdToken, + reload, } from "firebase/auth"; import { Provider } from "config"; @@ -144,7 +146,13 @@ export class GoogleAuthService { } async confirmEmailVerify(code: string): Promise { - return applyActionCode(this.auth, code); + await applyActionCode(this.auth, code); + + // Reload the user and get a fresh token with email_verified: true + if (this.auth.currentUser) { + await reload(this.auth.currentUser); + await getIdToken(this.auth.currentUser, true); + } } async signInWithEmailLink(email: string): Promise { diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss index a756f2ddb779..997a2399432e 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.module.scss @@ -3,7 +3,6 @@ .content { display: flex; flex-direction: row; - padding-bottom: variables.$spacing-lg; } .mainView { diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.module.scss deleted file mode 100644 index 17d92ee1b1e6..000000000000 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.buttonsContainer { - display: flex; -} diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx index 21030a199977..fa40c8a3aedb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx @@ -3,6 +3,8 @@ import { FormattedMessage } from "react-intl"; import { CellProps } from "react-table"; import { HeadTitle } from "components/common/HeadTitle"; +import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { Table } from "components/ui/Table"; import { Connector, ConnectorDefinition } from "core/domain/connector"; @@ -12,10 +14,9 @@ import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import ConnectorCell from "./ConnectorCell"; -import styles from "./ConnectorsView.module.scss"; import CreateConnector from "./CreateConnector"; import ImageCell from "./ImageCell"; -import { Block, FormContentTitle, Title } from "./PageComponents"; +import { FormContentTitle } from "./PageComponents"; import UpgradeAllButton from "./UpgradeAllButton"; import VersionCell from "./VersionCell"; @@ -125,7 +126,7 @@ const ConnectorsView: React.FC = ({ const renderHeaderControls = (section: "used" | "available") => ((section === "used" && usedConnectorsDefinitions.length > 0) || (section === "available" && usedConnectorsDefinitions.length === 0)) && ( -
+ {allowUploadCustomImage && } {(hasNewConnectorVersion || isUpdateSuccess) && allowUpdateConnectors && ( = ({ onUpdate={onUpdate} /> )} -
+ ); return ( @@ -143,31 +144,37 @@ const ConnectorsView: React.FC = ({ - {usedConnectorsDefinitions.length > 0 && ( - - - <FormattedMessage id={type === "sources" ? "admin.manageSource" : "admin.manageDestination"} /> - {renderHeaderControls("used")} - + + {usedConnectorsDefinitions.length > 0 && ( + + + + + + {renderHeaderControls("used")} + + + + )} + + + + + + + {renderHeaderControls("available")} +
- - )} - - - - <FormattedMessage id={type === "sources" ? "admin.availableSource" : "admin.availableDestinations"} /> - {renderHeaderControls("available")} - -
- + + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx index 31a9eaff246d..ef7827175dd4 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/PageComponents.tsx @@ -1,27 +1,11 @@ import styled from "styled-components"; -import { H5 } from "components/base/Titles"; - -const Title = styled(H5)` - color: ${({ theme }) => theme.darkPrimaryColor}; - margin-bottom: 19px; - display: flex; - justify-content: space-between; - align-items: center; -`; - -const Block = styled.div` - margin-bottom: 56px; -`; - -const FormContent = styled.div` +export const FormContent = styled.div` width: 253px; margin: -10px 0 -10px 200px; position: relative; `; -const FormContentTitle = styled(FormContent)` +export const FormContentTitle = styled(FormContent)` margin: 0 0 0 200px; `; - -export { Title, Block, FormContent, FormContentTitle }; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss index fc965f61fed9..1bba26e32786 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.module.scss @@ -1,8 +1,6 @@ @use "scss/variables"; .content { - padding-bottom: variables.$spacing-md; - > form { display: flex; flex-direction: column; diff --git a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss index 48419e552f16..8badacf0e855 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.module.scss @@ -31,10 +31,6 @@ } } -.contentCard { - margin-bottom: variables.$spacing-xl; -} - .footer { width: 100%; display: flex; diff --git a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx index 29bcdb7986bb..c5f87cbf6a4b 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionStatusPage/ConnectionStatusPage.tsx @@ -159,7 +159,6 @@ export const ConnectionStatusPage: React.FC = () => { return ( <> diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss index 6903e64a247b..681bc32cc405 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.module.scss @@ -1,7 +1,17 @@ +@use "scss/variables"; + .content { max-width: 1073px; margin: 0 auto; - padding-bottom: 10px; + + & > fieldset { + margin: 0; + padding: 0; + border: 0; + display: flex; + flex-direction: column; + gap: variables.$spacing-lg; + } } .customCard { diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx index 17bd026c7c27..40c9ee19e2b9 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx @@ -61,10 +61,7 @@ export const ConnectionTransformationPage: React.FC = () => { return (
-
+
{supportsNormalization && } {supportsDbt && } {supportsCloudDbtIntegration && } diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx index 0b8a0b8455f3..88a58210773c 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx @@ -28,7 +28,6 @@ export const CustomTransformationsCard: React.FC<{ title={} collapsible - bottomSeparator form={{ initialValues, enableReinitialize: true, diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss index 09620779260b..9d9a85316dbc 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/DbtCloudTransformationsCard.module.scss @@ -5,7 +5,7 @@ display: flex; flex-direction: column; align-items: center; - padding: 25px 25px 22px; + padding: variables.$spacing-xl; background-color: colors.$grey-50; } diff --git a/airbyte-webapp/src/scss/_variables.scss b/airbyte-webapp/src/scss/_variables.scss index 5f5d103e38b0..19c967957ab5 100644 --- a/airbyte-webapp/src/scss/_variables.scss +++ b/airbyte-webapp/src/scss/_variables.scss @@ -26,7 +26,7 @@ $spacing-md: 10px; $spacing-lg: 15px; $spacing-xl: 20px; $spacing-2xl: 40px; -$spacing-page-bottom: 150px; +$spacing-page-bottom-cloud: 88px; $main-page-content-min-width: 960px; $width-size-menu: 93px; diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss index 626cae0e8051..41c28fbedb93 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss +++ b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss @@ -3,7 +3,7 @@ .leftPanel { > *:last-child { - padding-bottom: variables.$spacing-page-bottom; + padding-bottom: variables.$spacing-page-bottom-cloud; } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java index 295112266eca..2e321793c832 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java @@ -14,6 +14,7 @@ import static io.airbyte.persistence.job.models.AttemptStatus.FAILED; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import datadog.trace.api.Trace; import io.airbyte.commons.docker.DockerUtils; @@ -66,7 +67,6 @@ import jakarta.inject.Singleton; import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -483,10 +483,13 @@ private void failNonTerminalJobs(final UUID connectionId) { ReleaseStage.generally_available, 4); private static final Comparator RELEASE_STAGE_COMPARATOR = Comparator.comparingInt(RELEASE_STAGE_ORDER::get); - private static List orderByReleaseStageAsc(final List releaseStages) { - final List copiedList = new ArrayList<>(releaseStages); - copiedList.sort(RELEASE_STAGE_COMPARATOR); - return copiedList; + @VisibleForTesting + static List orderByReleaseStageAsc(final List releaseStages) { + // Using collector to get a mutable list + return releaseStages.stream() + .filter(stage -> stage != null) + .sorted(RELEASE_STAGE_COMPARATOR) + .toList(); } /** diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java index ccdb8447fd6c..0efcb27e5c6d 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java @@ -33,6 +33,7 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; import io.airbyte.persistence.job.JobCreator; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; @@ -535,4 +536,13 @@ void ensureCleanJobState() throws IOException { } + @Test + void testReleaseStageOrdering() { + final List input = List.of(ReleaseStage.alpha, ReleaseStage.custom, ReleaseStage.beta, ReleaseStage.generally_available); + final List expected = List.of(ReleaseStage.custom, ReleaseStage.alpha, ReleaseStage.beta, ReleaseStage.generally_available); + + Assertions.assertThat(JobCreationAndStatusUpdateActivityImpl.orderByReleaseStageAsc(input)) + .containsExactlyElementsOf(expected); + } + } diff --git a/deps.toml b/deps.toml index f3a642d9bab4..0fa2bd639d69 100644 --- a/deps.toml +++ b/deps.toml @@ -24,6 +24,7 @@ micronaut = "3.8.1" micronaut-test = "3.8.0" platform-testcontainers = "1.17.3" postgresql = "42.3.5" +reactor = "3.5.2" slf4j = "1.7.36" temporal = "1.17.0" @@ -101,6 +102,7 @@ platform-testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref platform-testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "platform-testcontainers" } postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" } quartz-scheduler = { module = "org.quartz-scheduler:quartz", version = "2.3.2" } +reactor-core = { module = "io.projectreactor:reactor-core", version.ref = "reactor" } s3 = { module = "software.amazon.awssdk:s3", version = "2.16.84" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version = "4.7.3" } diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index e418918e1f78..6b6759b152f5 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -136,6 +136,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| +| 1.2.12 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 1.2.11 | 2023-01-18 | [#21144](https://github.com/airbytehq/airbyte/pull/21144) | Added explicit error message if sync fails due to a config issue | | 1.2.9 | 2022-12-14 | [#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check | | 1.2.8 | 2022-11-22 | [#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index aa07d098ed17..e988530da08c 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -235,31 +235,32 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.2.12 | 2022-10-18 | [\#17901](https://github.com/airbytehq/airbyte/pull/17901) | Fix logging to GCS | -| 0.2.11 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | -| 0.2.10 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 0.2.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | -| 0.2.8 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | -| 0.2.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | -| 0.2.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | -| 0.2.5 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | -| 0.2.4 | 2022-04-22 | [\#12167](https://github.com/airbytehq/airbyte/pull/12167) | Add gzip compression option for CSV and JSONL formats. | -| 0.2.3 | 2022-04-22 | [\#11795](https://github.com/airbytehq/airbyte/pull/11795) | Fix the connection check to verify the provided bucket path. | -| 0.2.2 | 2022-04-05 | [\#11728](https://github.com/airbytehq/airbyte/pull/11728) | Properly clean-up bucket when running OVERWRITE sync mode | -| 0.2.1 | 2022-04-05 | [\#11499](https://github.com/airbytehq/airbyte/pull/11499) | Updated spec and documentation. | -| 0.2.0 | 2022-04-04 | [\#11686](https://github.com/airbytehq/airbyte/pull/11686) | Use serialized buffering strategy to reduce memory consumption; compress CSV and JSONL formats. | -| 0.1.22 | 2022-02-12 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add JVM flag to exist on OOME. | -| 0.1.21 | 2022-02-12 | [\#10299](https://github.com/airbytehq/airbyte/pull/10299) | Fix connection check to require only the necessary permissions. | -| 0.1.20 | 2022-01-11 | [\#9367](https://github.com/airbytehq/airbyte/pull/9367) | Avro & Parquet: support array field with unknown item type; default any improperly typed field to string. | -| 0.1.19 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | -| 0.1.18 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | -| 0.1.17 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types | -| 0.1.16 | 2021-12-20 | [\#8974](https://github.com/airbytehq/airbyte/pull/8974) | Release a new version to ensure there is no excessive logging. | -| 0.1.15 | 2021-12-03 | [\#8386](https://github.com/airbytehq/airbyte/pull/8386) | Add new GCP regions | -| 0.1.14 | 2021-12-01 | [\#7732](https://github.com/airbytehq/airbyte/pull/7732) | Support timestamp in Avro and Parquet | -| 0.1.13 | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288) | Support Json `additionalProperties`. | -| 0.1.2 | 2021-09-12 | [\#5720](https://github.com/airbytehq/airbyte/issues/5720) | Added configurable block size for stream. Each stream is limited to 10,000 by GCS | -| 0.1.1 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added storing gcsCsvFileLocation property for CSV format. This is used by destination-bigquery \(GCS Staging upload type\) | -| 0.1.0 | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784) | Initial release. | +| Version | Date | Pull Request | Subject | +|:--------| :--- |:------------------------------------------------------------| :--- | +| 0.2.13 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | +| 0.2.12 | 2022-10-18 | [\#17901](https://github.com/airbytehq/airbyte/pull/17901) | Fix logging to GCS | +| 0.2.11 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | +| 0.2.10 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.2.9 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | +| 0.2.8 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | +| 0.2.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | +| 0.2.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | +| 0.2.5 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | +| 0.2.4 | 2022-04-22 | [\#12167](https://github.com/airbytehq/airbyte/pull/12167) | Add gzip compression option for CSV and JSONL formats. | +| 0.2.3 | 2022-04-22 | [\#11795](https://github.com/airbytehq/airbyte/pull/11795) | Fix the connection check to verify the provided bucket path. | +| 0.2.2 | 2022-04-05 | [\#11728](https://github.com/airbytehq/airbyte/pull/11728) | Properly clean-up bucket when running OVERWRITE sync mode | +| 0.2.1 | 2022-04-05 | [\#11499](https://github.com/airbytehq/airbyte/pull/11499) | Updated spec and documentation. | +| 0.2.0 | 2022-04-04 | [\#11686](https://github.com/airbytehq/airbyte/pull/11686) | Use serialized buffering strategy to reduce memory consumption; compress CSV and JSONL formats. | +| 0.1.22 | 2022-02-12 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add JVM flag to exist on OOME. | +| 0.1.21 | 2022-02-12 | [\#10299](https://github.com/airbytehq/airbyte/pull/10299) | Fix connection check to require only the necessary permissions. | +| 0.1.20 | 2022-01-11 | [\#9367](https://github.com/airbytehq/airbyte/pull/9367) | Avro & Parquet: support array field with unknown item type; default any improperly typed field to string. | +| 0.1.19 | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121) | Fixed check method for GCS mode to verify if all roles assigned to user | +| 0.1.18 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | +| 0.1.17 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types | +| 0.1.16 | 2021-12-20 | [\#8974](https://github.com/airbytehq/airbyte/pull/8974) | Release a new version to ensure there is no excessive logging. | +| 0.1.15 | 2021-12-03 | [\#8386](https://github.com/airbytehq/airbyte/pull/8386) | Add new GCP regions | +| 0.1.14 | 2021-12-01 | [\#7732](https://github.com/airbytehq/airbyte/pull/7732) | Support timestamp in Avro and Parquet | +| 0.1.13 | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288) | Support Json `additionalProperties`. | +| 0.1.2 | 2021-09-12 | [\#5720](https://github.com/airbytehq/airbyte/issues/5720) | Added configurable block size for stream. Each stream is limited to 10,000 by GCS | +| 0.1.1 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added storing gcsCsvFileLocation property for CSV format. This is used by destination-bigquery \(GCS Staging upload type\) | +| 0.1.0 | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784) | Initial release. | diff --git a/docs/integrations/destinations/google-sheets.md b/docs/integrations/destinations/google-sheets.md index ef4c122966ae..7b836e053508 100644 --- a/docs/integrations/destinations/google-sheets.md +++ b/docs/integrations/destinations/google-sheets.md @@ -107,12 +107,12 @@ Please be aware of the [Google Spreadsheet limitations](#limitations) before you ### Google Sheets Limitations -During the upload process and from the data storage perspective there are some limitations that should be considered beforehands: +During the upload process and from the data storage perspective there are some limitations that should be considered beforehand as [determined by Google here](https://support.google.com/drive/answer/37603): -- **Maximum of 5 Million Cells** +- **Maximum of 10 Million Cells** -A Google Sheets document can have a maximum of 5 million cells. These can be in a single worksheet or in multiple sheets. -In case you already have the 5 million limit reached in fewer columns, it will not allow you to add more columns (and vice versa, i.e., if 5 million cells limit is reached with a certain number of rows, it will not allow more rows). +A Google Sheets document can have a maximum of 10 million cells. These can be in a single worksheet or in multiple sheets. +In case you already have the 10 million limit reached in fewer columns, it will not allow you to add more columns (and vice versa, i.e., if 10 million cells limit is reached with a certain number of rows, it will not allow more rows). - **Maximum of 18,278 Columns** diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index a2a7168dcd03..0f2b0ac8e50f 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -141,6 +141,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.54 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 0.3.53 | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field | | 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers | | 0.3.51 | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling | @@ -178,5 +179,3 @@ Each stream will be output into its own raw table in Redshift. Each table will c | 0.3.13 | 2021-09-02 | [5745](https://github.com/airbytehq/airbyte/pull/5745) | Disable STATUPDATE flag when using S3 staging to speed up performance | | 0.3.12 | 2021-07-21 | [3555](https://github.com/airbytehq/airbyte/pull/3555) | Enable partial checkpointing for halfway syncs | | 0.3.11 | 2021-07-20 | [4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | - - diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index c642624c6ea3..db26ac102e0d 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -330,6 +330,7 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.19 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 0.3.18 | 2022-12-15 | [\#20088](https://github.com/airbytehq/airbyte/pull/20088) | New data type support v0/v1 | | 0.3.17 | 2022-10-15 | [\#18031](https://github.com/airbytehq/airbyte/pull/18031) | Fix integration tests to use bucket path | | 0.3.16 | 2022-10-03 | [\#17340](https://github.com/airbytehq/airbyte/pull/17340) | Enforced encrypted only traffic to S3 buckets and check logic | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index b9c6ba290ce2..1bbadee5053a 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -277,7 +277,8 @@ Now that you have set up the Snowflake destination connector, check out the foll | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.43 | 2023-01-20 | [\#21450](https://github.com/airbytehq/airbyte/pull/21450) | Updated Check methods to handle more possible s3 and gcs stagings issues | +| 0.4.44 | 2023-01-20 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | +| 0.4.43 | 2023-01-20 | [\#21450](https://github.com/airbytehq/airbyte/pull/21450) | Updated Check methods to handle more possible s3 and gcs stagings issues | | 0.4.42 | 2023-01-12 | [\#21342](https://github.com/airbytehq/airbyte/pull/21342) | Better handling for conflicting destination streams | | 0.4.41 | 2022-12-16 | [\#20566](https://github.com/airbytehq/airbyte/pull/20566) | Improve spec to adhere to standards | | 0.4.40 | 2022-11-11 | [\#19302](https://github.com/airbytehq/airbyte/pull/19302) | Set jdbc application env variable depends on env - airbyte_oss or airbyte_cloud | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 44a86a62e01c..eaadcae5f40b 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -341,7 +341,8 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.27 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| 0.4.28 | 2023-01-18 | [21348](https://github.com/airbytehq/airbyte/pull/21348) | Fix error introduced in [18959](https://github.com/airbytehq/airbyte/pull/18959) in which option `initial_waiting_seconds` was removed | +| 0.4.27 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 0.4.26 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | | 0.4.25 | 2022-11-04 | [18732](https://github.com/airbytehq/airbyte/pull/18732) | Upgrade debezium version to 1.9.6 | | 0.4.24 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 4b3ed0421585..b46e07971730 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -409,124 +409,125 @@ The root causes is that the WALs needed for the incremental sync has been remove ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1.0.38 | 2022-01-17 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | -| 1.0.37 | 2023-01-17 | [20783](https://github.com/airbytehq/airbyte/pull/20783) | Emit estimate trace messages for non-CDC mode. | -| 1.0.36 | 2023-01-11 | [21003](https://github.com/airbytehq/airbyte/pull/21003) | Handle null values for array data types in CDC mode gracefully. | -| 1.0.35 | 2023-01-04 | [20469](https://github.com/airbytehq/airbyte/pull/20469) | Introduce feature to make LSN commit behaviour configurable. | -| 1.0.34 | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378) | Improve descriptions | -| 1.0.33 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | -| 1.0.32 | 2022-12-12 | [20192](https://github.com/airbytehq/airbyte/pull/20192) | Only throw a warning if cursor column contains null values. | -| 1.0.31 | 2022-12-02 | [19889](https://github.com/airbytehq/airbyte/pull/19889) | Check before each sync and stop if an incremental sync cursor column contains a null value. | -| | 2022-12-02 | [19985](https://github.com/airbytehq/airbyte/pull/19985) | Reenable incorrectly-disabled `wal2json` CDC plugin | -| 1.0.30 | 2022-11-29 | [19024](https://github.com/airbytehq/airbyte/pull/19024) | Skip tables from schema where user do not have Usage permission during discovery. | -| 1.0.29 | 2022-11-29 | [19623](https://github.com/airbytehq/airbyte/pull/19623) | Mark PSQLException related to using replica that is configured as a hot standby server as config error. | -| 1.0.28 | 2022-11-28 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. | -| 1.0.27 | 2022-11-28 | [16990](https://github.com/airbytehq/airbyte/pull/16990) | Handle arrays data types | -| 1.0.26 | 2022-11-18 | [19551](https://github.com/airbytehq/airbyte/pull/19551) | Fixes bug with ssl modes | -| 1.0.25 | 2022-11-16 | [19004](https://github.com/airbytehq/airbyte/pull/19004) | Use Debezium heartbeats to improve CDC replication of large databases. | -| 1.0.24 | 2022-11-07 | [19291](https://github.com/airbytehq/airbyte/pull/19291) | Default timeout is reduced from 1 min to 10sec | -| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | -| 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | -| 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | -| 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | -| 1.0.19 | 2022-10-21 | [18263](https://github.com/airbytehq/airbyte/pull/18263) | Fixes bug introduced in [15833](https://github.com/airbytehq/airbyte/pull/15833) and adds better error messaging for SSH tunnel in Destinations | -| 1.0.18 | 2022-10-19 | [18087](https://github.com/airbytehq/airbyte/pull/18087) | Better error messaging for configuration errors (SSH configs, choosing an invalid cursor) | -| 1.0.17 | 2022-10-17 | [18041](https://github.com/airbytehq/airbyte/pull/18041) | Fixes bug introduced 2022-09-12 with SshTunnel, handles iterator exception properly | -| 1.0.16 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 1.0.15 | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Handle 24:00:00 value for Time column | -| 1.0.14 | 2022-10-03 | [17515](https://github.com/airbytehq/airbyte/pull/17515) | Fix an issue preventing connection using client certificate | -| 1.0.13 | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459) | Upgrade debezium version to 1.9.6 from 1.9.2 | -| 1.0.12 | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres source | -| 1.0.11 | 2022-09-26 | [17131](https://github.com/airbytehq/airbyte/pull/17131) | Allow nullable columns to be used as cursor | -| 1.0.10 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | -| 1.0.9 | 2022-09-13 | [16657](https://github.com/airbytehq/airbyte/pull/16657) | Improve CDC record queueing performance | -| 1.0.8 | 2022-09-08 | [16202](https://github.com/airbytehq/airbyte/pull/16202) | Adds error messaging factory to UI | -| 1.0.7 | 2022-08-30 | [16114](https://github.com/airbytehq/airbyte/pull/16114) | Prevent traffic going on an unsecured channel in strict-encryption version of source postgres | -| 1.0.6 | 2022-08-30 | [16138](https://github.com/airbytehq/airbyte/pull/16138) | Remove unnecessary logging | -| 1.0.5 | 2022-08-25 | [15993](https://github.com/airbytehq/airbyte/pull/15993) | Add support for connection over SSL in CDC mode | -| 1.0.4 | 2022-08-23 | [15877](https://github.com/airbytehq/airbyte/pull/15877) | Fix temporal data type bug which was causing failure in CDC mode | -| 1.0.3 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 1.0.2 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | -| 1.0.1 | 2022-08-10 | [15496](https://github.com/airbytehq/airbyte/pull/15496) | Fix state emission in incremental sync | -| | 2022-08-10 | [15481](https://github.com/airbytehq/airbyte/pull/15481) | Fix data handling from WAL logs in CDC mode | -| 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available (requires [upgrading](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to `v0.40.0-alpha`) | -| 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | -| 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | -| 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | -| 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | -| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) | -| 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | -| | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | -| 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | -| 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | -| 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. | -| 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters | -| 0.4.32 | 2022-07-07 | [14694](https://github.com/airbytehq/airbyte/pull/14694) | Force to produce LEGACY state if the use stream capable feature flag is set to false | -| 0.4.31 | 2022-07-07 | [14447](https://github.com/airbytehq/airbyte/pull/14447) | Under CDC mode, retrieve only those tables included in the publications | -| 0.4.30 | 2022-06-30 | [14251](https://github.com/airbytehq/airbyte/pull/14251) | Use more simple and comprehensive query to get selectable tables | -| 0.4.29 | 2022-06-29 | [14265](https://github.com/airbytehq/airbyte/pull/14265) | Upgrade postgresql JDBC version to 42.3.5 | -| 0.4.28 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | -| 0.4.26 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.4.25 | 2022-06-15 | [13823](https://github.com/airbytehq/airbyte/pull/13823) | Publish adaptive postgres source that enforces ssl on cloud + Debezium version upgrade to 1.9.2 from 1.4.2 | -| 0.4.24 | 2022-06-14 | [13549](https://github.com/airbytehq/airbyte/pull/13549) | Fixed truncated precision if the value of microseconds or seconds is 0 | -| 0.4.23 | 2022-06-13 | [13655](https://github.com/airbytehq/airbyte/pull/13745) | Fixed handling datetime cursors when upgrading from older versions of the connector | -| 0.4.22 | 2022-06-09 | [13655](https://github.com/airbytehq/airbyte/pull/13655) | Fixed bug with unsupported date-time datatypes during incremental sync | -| 0.4.21 | 2022-06-06 | [13435](https://github.com/airbytehq/airbyte/pull/13435) | Adjust JDBC fetch size based on max memory and max row size | -| 0.4.20 | 2022-06-02 | [13367](https://github.com/airbytehq/airbyte/pull/13367) | Added convertion hstore to json format | -| 0.4.19 | 2022-05-25 | [13166](https://github.com/airbytehq/airbyte/pull/13166) | Added timezone awareness and handle BC dates | -| 0.4.18 | 2022-05-25 | [13083](https://github.com/airbytehq/airbyte/pull/13083) | Add support for tsquey type | -| 0.4.17 | 2022-05-19 | [13016](https://github.com/airbytehq/airbyte/pull/13016) | CDC modify schema to allow null values | -| 0.4.16 | 2022-05-14 | [12840](https://github.com/airbytehq/airbyte/pull/12840) | Added custom JDBC parameters field | -| 0.4.15 | 2022-05-13 | [12834](https://github.com/airbytehq/airbyte/pull/12834) | Fix the bug that the connector returns empty catalog for Azure Postgres database | -| 0.4.14 | 2022-05-08 | [12689](https://github.com/airbytehq/airbyte/pull/12689) | Add table retrieval according to role-based `SELECT` privilege | -| 0.4.13 | 2022-05-05 | [10230](https://github.com/airbytehq/airbyte/pull/10230) | Explicitly set null value for field in json | -| 0.4.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.4.11 | 2022-04-11 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.4.10 | 2022-04-08 | [11798](https://github.com/airbytehq/airbyte/pull/11798) | Fixed roles for fetching materialized view processing | -| 0.4.8 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.4.7 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.4.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.4.5 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | -| 0.4.4 | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description | -| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | -| 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | -| 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | -| 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types | -| 0.3.17 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.3.16 | 2021-11-28 | [7995](https://github.com/airbytehq/airbyte/pull/7995) | Fixed money type with amount > 1000 | -| 0.3.15 | 2021-11-26 | [8066](https://github.com/airbytehq/airbyte/pull/8266) | Fixed the case, when Views are not listed during schema discovery | -| 0.3.14 | 2021-11-17 | [8010](https://github.com/airbytehq/airbyte/pull/8010) | Added checking of privileges before table internal discovery | -| 0.3.13 | 2021-10-26 | [7339](https://github.com/airbytehq/airbyte/pull/7339) | Support or improve support for Interval, Money, Date, various geometric data types, inventory_items, and others | -| 0.3.12 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | -| 0.3.11 | 2021-09-02 | [5742](https://github.com/airbytehq/airbyte/pull/5742) | Add SSH Tunnel support | -| 0.3.9 | 2021-08-17 | [5304](https://github.com/airbytehq/airbyte/pull/5304) | Fix CDC OOM issue | -| 0.3.8 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | -| 0.3.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.3.3 | 2021-06-08 | [3960](https://github.com/airbytehq/airbyte/pull/3960) | Add method field in specification parameters | -| 0.3.2 | 2021-05-26 | [3179](https://github.com/airbytehq/airbyte/pull/3179) | Remove `isCDC` logging | -| 0.3.1 | 2021-04-21 | [2878](https://github.com/airbytehq/airbyte/pull/2878) | Set defined cursor for CDC | -| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | -| 0.2.7 | 2021-04-16 | [2923](https://github.com/airbytehq/airbyte/pull/2923) | SSL spec as optional | -| 0.2.6 | 2021-04-16 | [2757](https://github.com/airbytehq/airbyte/pull/2757) | Support SSL connection | -| 0.2.5 | 2021-04-12 | [2859](https://github.com/airbytehq/airbyte/pull/2859) | CDC bugfix | -| 0.2.4 | 2021-04-09 | [2548](https://github.com/airbytehq/airbyte/pull/2548) | Support CDC | -| 0.2.3 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | -| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | -| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.13 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | -| 0.1.12 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | -| 0.1.11 | 2021-01-25 | [1765](https://github.com/airbytehq/airbyte/pull/1765) | Add field titles to specification | -| 0.1.10 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | -| 0.1.9 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | -| 0.1.8 | 2021-01-13 | [1588](https://github.com/airbytehq/airbyte/pull/1588) | Handle invalid numeric values in JDBC source | -| 0.1.7 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySql to use new JdbcSource | -| 0.1.6 | 2020-12-09 | [1172](https://github.com/airbytehq/airbyte/pull/1172) | Support incremental sync | -| 0.1.5 | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038) | Change JDBC sources to discover more than standard schemas | -| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.39 | 2022-01-20 | [21683](https://github.com/airbytehq/airbyte/pull/21683) | Speed up esmtimates for trace messages in non-CDC mode. | +| 1.0.38 | 2022-01-17 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| 1.0.37 | 2023-01-17 | [20783](https://github.com/airbytehq/airbyte/pull/20783) | Emit estimate trace messages for non-CDC mode. | +| 1.0.36 | 2023-01-11 | [21003](https://github.com/airbytehq/airbyte/pull/21003) | Handle null values for array data types in CDC mode gracefully. | +| 1.0.35 | 2023-01-04 | [20469](https://github.com/airbytehq/airbyte/pull/20469) | Introduce feature to make LSN commit behaviour configurable. | +| 1.0.34 | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378) | Improve descriptions | +| 1.0.33 | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959) | CDC : Don't timeout if snapshot is not complete. | +| 1.0.32 | 2022-12-12 | [20192](https://github.com/airbytehq/airbyte/pull/20192) | Only throw a warning if cursor column contains null values. | +| 1.0.31 | 2022-12-02 | [19889](https://github.com/airbytehq/airbyte/pull/19889) | Check before each sync and stop if an incremental sync cursor column contains a null value. | +| | 2022-12-02 | [19985](https://github.com/airbytehq/airbyte/pull/19985) | Reenable incorrectly-disabled `wal2json` CDC plugin | +| 1.0.30 | 2022-11-29 | [19024](https://github.com/airbytehq/airbyte/pull/19024) | Skip tables from schema where user do not have Usage permission during discovery. | +| 1.0.29 | 2022-11-29 | [19623](https://github.com/airbytehq/airbyte/pull/19623) | Mark PSQLException related to using replica that is configured as a hot standby server as config error. | +| 1.0.28 | 2022-11-28 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. | +| 1.0.27 | 2022-11-28 | [16990](https://github.com/airbytehq/airbyte/pull/16990) | Handle arrays data types | +| 1.0.26 | 2022-11-18 | [19551](https://github.com/airbytehq/airbyte/pull/19551) | Fixes bug with ssl modes | +| 1.0.25 | 2022-11-16 | [19004](https://github.com/airbytehq/airbyte/pull/19004) | Use Debezium heartbeats to improve CDC replication of large databases. | +| 1.0.24 | 2022-11-07 | [19291](https://github.com/airbytehq/airbyte/pull/19291) | Default timeout is reduced from 1 min to 10sec | +| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | +| 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | +| 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | +| 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages | +| 1.0.19 | 2022-10-21 | [18263](https://github.com/airbytehq/airbyte/pull/18263) | Fixes bug introduced in [15833](https://github.com/airbytehq/airbyte/pull/15833) and adds better error messaging for SSH tunnel in Destinations | +| 1.0.18 | 2022-10-19 | [18087](https://github.com/airbytehq/airbyte/pull/18087) | Better error messaging for configuration errors (SSH configs, choosing an invalid cursor) | +| 1.0.17 | 2022-10-17 | [18041](https://github.com/airbytehq/airbyte/pull/18041) | Fixes bug introduced 2022-09-12 with SshTunnel, handles iterator exception properly | +| 1.0.16 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | +| 1.0.15 | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Handle 24:00:00 value for Time column | +| 1.0.14 | 2022-10-03 | [17515](https://github.com/airbytehq/airbyte/pull/17515) | Fix an issue preventing connection using client certificate | +| 1.0.13 | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459) | Upgrade debezium version to 1.9.6 from 1.9.2 | +| 1.0.12 | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres source | +| 1.0.11 | 2022-09-26 | [17131](https://github.com/airbytehq/airbyte/pull/17131) | Allow nullable columns to be used as cursor | +| 1.0.10 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | +| 1.0.9 | 2022-09-13 | [16657](https://github.com/airbytehq/airbyte/pull/16657) | Improve CDC record queueing performance | +| 1.0.8 | 2022-09-08 | [16202](https://github.com/airbytehq/airbyte/pull/16202) | Adds error messaging factory to UI | +| 1.0.7 | 2022-08-30 | [16114](https://github.com/airbytehq/airbyte/pull/16114) | Prevent traffic going on an unsecured channel in strict-encryption version of source postgres | +| 1.0.6 | 2022-08-30 | [16138](https://github.com/airbytehq/airbyte/pull/16138) | Remove unnecessary logging | +| 1.0.5 | 2022-08-25 | [15993](https://github.com/airbytehq/airbyte/pull/15993) | Add support for connection over SSL in CDC mode | +| 1.0.4 | 2022-08-23 | [15877](https://github.com/airbytehq/airbyte/pull/15877) | Fix temporal data type bug which was causing failure in CDC mode | +| 1.0.3 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 1.0.2 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | +| 1.0.1 | 2022-08-10 | [15496](https://github.com/airbytehq/airbyte/pull/15496) | Fix state emission in incremental sync | +| | 2022-08-10 | [15481](https://github.com/airbytehq/airbyte/pull/15481) | Fix data handling from WAL logs in CDC mode | +| 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available (requires [upgrading](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to `v0.40.0-alpha`) | +| 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | +| 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | +| 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | +| 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | +| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) | +| 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | +| | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | +| 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | +| 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | +| 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. | +| 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters | +| 0.4.32 | 2022-07-07 | [14694](https://github.com/airbytehq/airbyte/pull/14694) | Force to produce LEGACY state if the use stream capable feature flag is set to false | +| 0.4.31 | 2022-07-07 | [14447](https://github.com/airbytehq/airbyte/pull/14447) | Under CDC mode, retrieve only those tables included in the publications | +| 0.4.30 | 2022-06-30 | [14251](https://github.com/airbytehq/airbyte/pull/14251) | Use more simple and comprehensive query to get selectable tables | +| 0.4.29 | 2022-06-29 | [14265](https://github.com/airbytehq/airbyte/pull/14265) | Upgrade postgresql JDBC version to 42.3.5 | +| 0.4.28 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | +| 0.4.26 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.4.25 | 2022-06-15 | [13823](https://github.com/airbytehq/airbyte/pull/13823) | Publish adaptive postgres source that enforces ssl on cloud + Debezium version upgrade to 1.9.2 from 1.4.2 | +| 0.4.24 | 2022-06-14 | [13549](https://github.com/airbytehq/airbyte/pull/13549) | Fixed truncated precision if the value of microseconds or seconds is 0 | +| 0.4.23 | 2022-06-13 | [13655](https://github.com/airbytehq/airbyte/pull/13745) | Fixed handling datetime cursors when upgrading from older versions of the connector | +| 0.4.22 | 2022-06-09 | [13655](https://github.com/airbytehq/airbyte/pull/13655) | Fixed bug with unsupported date-time datatypes during incremental sync | +| 0.4.21 | 2022-06-06 | [13435](https://github.com/airbytehq/airbyte/pull/13435) | Adjust JDBC fetch size based on max memory and max row size | +| 0.4.20 | 2022-06-02 | [13367](https://github.com/airbytehq/airbyte/pull/13367) | Added convertion hstore to json format | +| 0.4.19 | 2022-05-25 | [13166](https://github.com/airbytehq/airbyte/pull/13166) | Added timezone awareness and handle BC dates | +| 0.4.18 | 2022-05-25 | [13083](https://github.com/airbytehq/airbyte/pull/13083) | Add support for tsquey type | +| 0.4.17 | 2022-05-19 | [13016](https://github.com/airbytehq/airbyte/pull/13016) | CDC modify schema to allow null values | +| 0.4.16 | 2022-05-14 | [12840](https://github.com/airbytehq/airbyte/pull/12840) | Added custom JDBC parameters field | +| 0.4.15 | 2022-05-13 | [12834](https://github.com/airbytehq/airbyte/pull/12834) | Fix the bug that the connector returns empty catalog for Azure Postgres database | +| 0.4.14 | 2022-05-08 | [12689](https://github.com/airbytehq/airbyte/pull/12689) | Add table retrieval according to role-based `SELECT` privilege | +| 0.4.13 | 2022-05-05 | [10230](https://github.com/airbytehq/airbyte/pull/10230) | Explicitly set null value for field in json | +| 0.4.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.4.11 | 2022-04-11 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.4.10 | 2022-04-08 | [11798](https://github.com/airbytehq/airbyte/pull/11798) | Fixed roles for fetching materialized view processing | +| 0.4.8 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.4.7 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.4.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.4.5 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | +| 0.4.4 | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description | +| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | +| 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | +| 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | +| 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types | +| 0.3.17 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.3.16 | 2021-11-28 | [7995](https://github.com/airbytehq/airbyte/pull/7995) | Fixed money type with amount > 1000 | +| 0.3.15 | 2021-11-26 | [8066](https://github.com/airbytehq/airbyte/pull/8266) | Fixed the case, when Views are not listed during schema discovery | +| 0.3.14 | 2021-11-17 | [8010](https://github.com/airbytehq/airbyte/pull/8010) | Added checking of privileges before table internal discovery | +| 0.3.13 | 2021-10-26 | [7339](https://github.com/airbytehq/airbyte/pull/7339) | Support or improve support for Interval, Money, Date, various geometric data types, inventory_items, and others | +| 0.3.12 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | +| 0.3.11 | 2021-09-02 | [5742](https://github.com/airbytehq/airbyte/pull/5742) | Add SSH Tunnel support | +| 0.3.9 | 2021-08-17 | [5304](https://github.com/airbytehq/airbyte/pull/5304) | Fix CDC OOM issue | +| 0.3.8 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| 0.3.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.3.3 | 2021-06-08 | [3960](https://github.com/airbytehq/airbyte/pull/3960) | Add method field in specification parameters | +| 0.3.2 | 2021-05-26 | [3179](https://github.com/airbytehq/airbyte/pull/3179) | Remove `isCDC` logging | +| 0.3.1 | 2021-04-21 | [2878](https://github.com/airbytehq/airbyte/pull/2878) | Set defined cursor for CDC | +| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | +| 0.2.7 | 2021-04-16 | [2923](https://github.com/airbytehq/airbyte/pull/2923) | SSL spec as optional | +| 0.2.6 | 2021-04-16 | [2757](https://github.com/airbytehq/airbyte/pull/2757) | Support SSL connection | +| 0.2.5 | 2021-04-12 | [2859](https://github.com/airbytehq/airbyte/pull/2859) | CDC bugfix | +| 0.2.4 | 2021-04-09 | [2548](https://github.com/airbytehq/airbyte/pull/2548) | Support CDC | +| 0.2.3 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | +| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | +| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.13 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | +| 0.1.12 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | +| 0.1.11 | 2021-01-25 | [1765](https://github.com/airbytehq/airbyte/pull/1765) | Add field titles to specification | +| 0.1.10 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | +| 0.1.9 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | +| 0.1.8 | 2021-01-13 | [1588](https://github.com/airbytehq/airbyte/pull/1588) | Handle invalid numeric values in JDBC source | +| 0.1.7 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySql to use new JdbcSource | +| 0.1.6 | 2020-12-09 | [1172](https://github.com/airbytehq/airbyte/pull/1172) | Support incremental sync | +| 0.1.5 | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038) | Change JDBC sources to discover more than standard schemas | +| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | diff --git a/tools/bin/ci_integration_test.sh b/tools/bin/ci_integration_test.sh index 8fb208fdf83a..3fbd1a65970d 100755 --- a/tools/bin/ci_integration_test.sh +++ b/tools/bin/ci_integration_test.sh @@ -32,6 +32,7 @@ else export SUB_BUILD="CONNECTORS_BASE" elif [[ "$connector" == *"connectors"* ]]; then connector_name=$(echo $connector | cut -d / -f 2) + run-qa-checks $connector_name selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector_name$" || echo "") integrationTestCommand="$(_to_gradle_path "airbyte-integrations/$connector" integrationTest)" else diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_checks.py b/tools/ci_connector_ops/ci_connector_ops/qa_checks.py new file mode 100644 index 000000000000..dc384641c454 --- /dev/null +++ b/tools/ci_connector_ops/ci_connector_ops/qa_checks.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys + +def check_documentation_markdown_file(connector_name: str) -> bool: + """Check if a markdown file with connector documentation is available + in docs/integrations//.md + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether a documentation file was found. + """ + # TODO implement + return True + +def check_changelog_entry_is_updated(connector_name: str) -> bool: + """Check that the changelog entry is updated for the latest connector version + in docs/integrations//.md + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether a the changelog is up to date. + """ + # TODO implement + return True + +def check_connector_icon_is_available(connector_name: str) -> bool: + """Check an SVG icon exists for a connector in + in airbyte-config/init/src/main/resources/icons/.svg + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether an icon exists for this connector. + """ + # TODO implement + return True + +def check_connector_https_url_only(connector_name: str) -> bool: + """Check a connector code contains only https url. + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether the connector code contains only https url. + """ + # TODO implement + return True + +def check_connector_has_no_critical_vulnerabilities(connector_name: str) -> bool: + """Check if the connector image is free of critical Snyk vulnerabilities. + Runs a docker scan command. + + Args: + connector_name (str): The connector name + + Returns: + bool: Wether the connector is free of critical vulnerabilities. + """ + # TODO implement + return True + +QA_CHECKS = [ + check_documentation_markdown_file, + check_changelog_entry_is_updated, + check_connector_icon_is_available, + check_connector_https_url_only, + check_connector_has_no_critical_vulnerabilities +] + +def run_qa_checks(): + connector_name = sys.argv[1] + print(f"Running QA checks for {connector_name}") + qa_check_results = {qa_check.__name__: qa_check(connector_name) for qa_check in QA_CHECKS} + if not all(qa_check_results.values()): + print(f"QA checks failed for {connector_name}") + for check_name, check_result in qa_check_results.items(): + check_result_prefix = "✅" if check_result else "❌" + print(f"{check_result_prefix} - {check_name}") + sys.exit(1) + else: + print(f"All QA checks succeeded for {connector_name}") + sys.exit(0) diff --git a/tools/ci_connector_ops/setup.py b/tools/ci_connector_ops/setup.py index 09928f800c24..c573c8d7148b 100644 --- a/tools/ci_connector_ops/setup.py +++ b/tools/ci_connector_ops/setup.py @@ -9,7 +9,7 @@ setup( - version="0.1.1", + version="0.1.2", name="ci_connector_ops", description="Packaged maintained by the connector operations team to perform CI for connectors", author="Airbyte", @@ -21,7 +21,8 @@ "console_scripts": [ "check-test-strictness-level = ci_connector_ops.sat_config_checks:check_test_strictness_level", "write-review-requirements-file = ci_connector_ops.sat_config_checks:write_review_requirements_file", - "print-mandatory-reviewers = ci_connector_ops.sat_config_checks:print_mandatory_reviewers" + "print-mandatory-reviewers = ci_connector_ops.sat_config_checks:print_mandatory_reviewers", + "run-qa-checks = ci_connector_ops.qa_checks:run_qa_checks" ], }, )