diff --git a/.github/workflows/connector_teams_review_requirements.yml b/.github/workflows/connector_teams_review_requirements.yml index 6b9d953704b9..e4755ae54a50 100644 --- a/.github/workflows/connector_teams_review_requirements.yml +++ b/.github/workflows/connector_teams_review_requirements.yml @@ -5,6 +5,8 @@ on: paths: - "airbyte-integrations/connectors/source-**" pull_request_review: + paths: + - "airbyte-integrations/connectors/source-**" jobs: check-review-requirements: name: "Check if a review is required from Connector teams" diff --git a/.github/workflows/deploy-docs-site.yml b/.github/workflows/deploy-docs-site.yml index a147a8af5b71..30e993f4c506 100644 --- a/.github/workflows/deploy-docs-site.yml +++ b/.github/workflows/deploy-docs-site.yml @@ -9,6 +9,7 @@ on: pull_request: types: + - closed - opened - reopened - synchronize @@ -42,7 +43,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: |- export SKIP_DEPLOY="yes" - if [ "${{github.event_name}}" = 'push' -o "${{github.event_name}}" = 'workflow_dispatch' ]; then + if [ "${{github.event_name}}" = 'push' -o "${{github.event_name}}" = 'workflow_dispatch' -o "${{github.event.pull_request.merged}}" = 'true']; then export SKIP_DEPLOY="no" fi diff --git a/airbyte-commons-converters/build.gradle b/airbyte-commons-converters/build.gradle new file mode 100644 index 000000000000..8f42a8c4b8a0 --- /dev/null +++ b/airbyte-commons-converters/build.gradle @@ -0,0 +1,30 @@ +plugins { + id "java-library" +} + +dependencies { + annotationProcessor platform(libs.micronaut.bom) + annotationProcessor libs.bundles.micronaut.annotation.processor + + implementation platform(libs.micronaut.bom) + implementation libs.bundles.micronaut + + implementation project(':airbyte-api') + implementation project(':airbyte-commons') + implementation project(':airbyte-commons-protocol') + implementation project(':airbyte-config:config-models') + implementation project(':airbyte-config:config-persistence') + implementation project(':airbyte-json-validation') + implementation project(':airbyte-persistence:job-persistence') + implementation project(':airbyte-protocol:protocol-models') + implementation libs.guava + implementation libs.slf4j.api + + testAnnotationProcessor platform(libs.micronaut.bom) + testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor + testAnnotationProcessor libs.jmh.annotations + + testImplementation libs.bundles.micronaut.test +} + +Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/CatalogClientConverters.java similarity index 96% rename from airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java rename to airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/CatalogClientConverters.java index 52cf8c1c2db1..36f76489c0f6 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java +++ b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/CatalogClientConverters.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -34,12 +34,12 @@ public class CatalogClientConverters { */ public static io.airbyte.protocol.models.AirbyteCatalog toAirbyteProtocol(final io.airbyte.api.client.model.generated.AirbyteCatalog catalog) { - io.airbyte.protocol.models.AirbyteCatalog protoCatalog = + final io.airbyte.protocol.models.AirbyteCatalog protoCatalog = new io.airbyte.protocol.models.AirbyteCatalog(); - var airbyteStream = catalog.getStreams().stream().map(stream -> { + final var airbyteStream = catalog.getStreams().stream().map(stream -> { try { return toConfiguredProtocol(stream.getStream(), stream.getConfig()); - } catch (JsonValidationException e) { + } catch (final JsonValidationException e) { return null; } }).collect(Collectors.toList()); @@ -50,7 +50,7 @@ public static io.airbyte.protocol.models.AirbyteCatalog toAirbyteProtocol(final @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private static io.airbyte.protocol.models.AirbyteStream toConfiguredProtocol(final io.airbyte.api.client.model.generated.AirbyteStream stream, - AirbyteStreamConfiguration config) + final AirbyteStreamConfiguration config) throws JsonValidationException { if (config.getFieldSelectionEnabled() != null && config.getFieldSelectionEnabled()) { // Validate the selected field paths. diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ConnectionHelper.java similarity index 99% rename from airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java rename to airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ConnectionHelper.java index 2017f2564240..acc00aeba2d0 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java +++ b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ConnectionHelper.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; import com.google.common.base.Preconditions; import io.airbyte.commons.enums.Enums; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ConnectorConfigUpdater.java similarity index 98% rename from airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java rename to airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ConnectorConfigUpdater.java index 440e659dcb3f..357e31cf901b 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java +++ b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ConnectorConfigUpdater.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; import com.google.common.hash.Hashing; import io.airbyte.api.client.AirbyteApiClient; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ProtocolConverters.java b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ProtocolConverters.java similarity index 97% rename from airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ProtocolConverters.java rename to airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ProtocolConverters.java index edccab0fecea..73139c3f319b 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ProtocolConverters.java +++ b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ProtocolConverters.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; import io.airbyte.api.model.generated.StreamDescriptor; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/StateConverter.java b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/StateConverter.java similarity index 99% rename from airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/StateConverter.java rename to airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/StateConverter.java index 5be2bf1f6a45..c69a49ba692b 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/StateConverter.java +++ b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/StateConverter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; import io.airbyte.api.model.generated.ConnectionState; import io.airbyte.api.model.generated.ConnectionStateType; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ThreadedTimeTracker.java b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ThreadedTimeTracker.java similarity index 97% rename from airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ThreadedTimeTracker.java rename to airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ThreadedTimeTracker.java index 258db80bbc62..67eaf80411c3 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/ThreadedTimeTracker.java +++ b/airbyte-commons-converters/src/main/java/io/airbyte/commons/converters/ThreadedTimeTracker.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; /** * This class exists to track timing information for the sync. It needs to be thread-safe as diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java b/airbyte-commons-converters/src/test/java/io/airbyte/commons/converters/CatalogClientConvertersTest.java similarity index 98% rename from airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java rename to airbyte-commons-converters/src/test/java/io/airbyte/commons/converters/CatalogClientConvertersTest.java index b2a009c5e643..7dbf318fbed3 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java +++ b/airbyte-commons-converters/src/test/java/io/airbyte/commons/converters/CatalogClientConvertersTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java b/airbyte-commons-converters/src/test/java/io/airbyte/commons/converters/ConnectorConfigUpdaterTest.java similarity index 98% rename from airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java rename to airbyte-commons-converters/src/test/java/io/airbyte/commons/converters/ConnectorConfigUpdaterTest.java index 77c1e408fad8..2a5020303d94 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java +++ b/airbyte-commons-converters/src/test/java/io/airbyte/commons/converters/ConnectorConfigUpdaterTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helper; +package io.airbyte.commons.converters; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; diff --git a/airbyte-commons-server/build.gradle b/airbyte-commons-server/build.gradle index ec25c650ae7a..e56c24679a7c 100644 --- a/airbyte-commons-server/build.gradle +++ b/airbyte-commons-server/build.gradle @@ -29,8 +29,8 @@ dependencies { implementation project(':airbyte-analytics') implementation project(':airbyte-api') + implementation project(':airbyte-commons-converters') implementation project(':airbyte-commons-temporal') - implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:init') implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java index 2f2b73d8b66b..d464a40cf6d1 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java @@ -20,6 +20,7 @@ import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig; import io.airbyte.api.model.generated.ResourceRequirements; +import io.airbyte.commons.converters.StateConverter; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.server.handlers.helpers.CatalogConverter; import io.airbyte.config.BasicSchedule; @@ -28,7 +29,6 @@ import io.airbyte.config.State; import io.airbyte.config.StateWrapper; import io.airbyte.config.helpers.StateMessageHelper; -import io.airbyte.workers.helper.StateConverter; import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/CatalogDiffConverters.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/CatalogDiffConverters.java index 77d763b0f555..b4c6351bbce6 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/CatalogDiffConverters.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/CatalogDiffConverters.java @@ -9,10 +9,10 @@ import io.airbyte.api.model.generated.FieldSchemaUpdate; import io.airbyte.api.model.generated.FieldTransform; import io.airbyte.api.model.generated.StreamTransform; +import io.airbyte.commons.converters.ProtocolConverters; import io.airbyte.commons.enums.Enums; import io.airbyte.protocol.models.transform_models.FieldTransformType; import io.airbyte.protocol.models.transform_models.StreamTransformType; -import io.airbyte.workers.helper.ProtocolConverters; import java.util.List; import java.util.Optional; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/JobConverter.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/JobConverter.java index 1a5e59f7d1a6..d8ce04516574 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/JobConverter.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/JobConverter.java @@ -27,6 +27,7 @@ import io.airbyte.api.model.generated.ResetConfig; import io.airbyte.api.model.generated.SourceDefinitionRead; import io.airbyte.api.model.generated.SynchronousJobRead; +import io.airbyte.commons.converters.ProtocolConverters; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.server.scheduler.SynchronousJobMetadata; import io.airbyte.commons.server.scheduler.SynchronousResponse; @@ -44,7 +45,6 @@ import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.AttemptNormalizationStatus; import io.airbyte.persistence.job.models.Job; -import io.airbyte.workers.helper.ProtocolConverters; import jakarta.inject.Singleton; import java.io.IOException; import java.nio.file.Path; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java index d159297329b9..c7a58519006b 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java @@ -24,6 +24,7 @@ import io.airbyte.api.model.generated.SourceSearch; import io.airbyte.api.model.generated.StreamDescriptor; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; +import io.airbyte.commons.converters.ConnectionHelper; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.converters.ApiPojoConverters; @@ -55,7 +56,6 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.validation.json.JsonValidationException; -import io.airbyte.workers.helper.ConnectionHelper; import jakarta.inject.Singleton; import java.io.IOException; import java.util.Collections; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobHistoryHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobHistoryHandler.java index e3a7d01665ef..6add07e42591 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobHistoryHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobHistoryHandler.java @@ -137,7 +137,7 @@ public JobReadList listJobsFor(final JobListRequestBody request) throws IOExcept for (final AttemptRead a : jwar.getAttempts()) { final var stat = stats.get(new JobAttemptPair(jwar.getJob().getId(), a.getId().intValue())); if (stat == null) { - log.error("Missing stats for job {} attempt {}", jwar.getJob().getId(), a.getId().intValue()); + log.warn("Missing stats for job {} attempt {}", jwar.getJob().getId(), a.getId().intValue()); continue; } @@ -197,7 +197,7 @@ public JobInfoLightRead getJobInfoLight(final JobIdRequestBody jobIdRequestBody) } public JobOptionalRead getLastReplicationJob(final ConnectionIdRequestBody connectionIdRequestBody) throws IOException { - Optional job = jobPersistence.getLastReplicationJob(connectionIdRequestBody.getConnectionId()); + final Optional job = jobPersistence.getLastReplicationJob(connectionIdRequestBody.getConnectionId()); if (job.isEmpty()) { return new JobOptionalRead(); } else { diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/StateHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/StateHandler.java index 7ca6709634aa..2127797e5561 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/StateHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/StateHandler.java @@ -7,9 +7,9 @@ import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ConnectionState; import io.airbyte.api.model.generated.ConnectionStateCreateOrUpdate; +import io.airbyte.commons.converters.StateConverter; import io.airbyte.config.StateWrapper; import io.airbyte.config.persistence.StatePersistence; -import io.airbyte.workers.helper.StateConverter; import jakarta.inject.Singleton; import java.io.IOException; import java.util.Optional; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java index b863e56cdedb..9a601aa679f7 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java @@ -46,6 +46,7 @@ import io.airbyte.api.model.generated.WebBackendOperationCreateOrUpdate; import io.airbyte.api.model.generated.WebBackendWorkspaceState; import io.airbyte.api.model.generated.WebBackendWorkspaceStateResult; +import io.airbyte.commons.converters.ProtocolConverters; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.MoreBooleans; @@ -61,7 +62,6 @@ import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.validation.json.JsonValidationException; -import io.airbyte.workers.helper.ProtocolConverters; import jakarta.inject.Singleton; import java.io.IOException; import java.util.ArrayList; diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java index d595489c6be0..1b0bea10a58f 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java @@ -41,6 +41,7 @@ import io.airbyte.api.model.generated.StreamDescriptor; import io.airbyte.api.model.generated.SyncMode; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; +import io.airbyte.commons.converters.ConnectionHelper; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.converters.ApiPojoConverters; @@ -70,7 +71,6 @@ import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.validation.json.JsonValidationException; -import io.airbyte.workers.helper.ConnectionHelper; import java.io.IOException; import java.util.Collections; import java.util.List; diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/StateHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/StateHandlerTest.java index 381924a7e84c..fd384bbfb516 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/StateHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/StateHandlerTest.java @@ -14,6 +14,7 @@ import io.airbyte.api.model.generated.ConnectionStateType; import io.airbyte.api.model.generated.GlobalState; import io.airbyte.api.model.generated.StreamState; +import io.airbyte.commons.converters.ProtocolConverters; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.config.StateType; @@ -24,7 +25,6 @@ import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.workers.helper.ProtocolConverters; import java.io.IOException; import java.util.List; import java.util.Optional; diff --git a/airbyte-commons-worker/build.gradle b/airbyte-commons-worker/build.gradle index 8b46f830a30d..b7709d93fceb 100644 --- a/airbyte-commons-worker/build.gradle +++ b/airbyte-commons-worker/build.gradle @@ -20,6 +20,7 @@ dependencies { implementation libs.bundles.datadog implementation project(':airbyte-api') + implementation project(':airbyte-commons-converters') implementation project(':airbyte-commons-protocol') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-config:config-models') diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java index 02788ec88344..79aeee24e755 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; @@ -26,7 +27,6 @@ import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.DefaultAirbyteStreamFactory; import io.airbyte.workers.process.IntegrationLauncher; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java index b0a5168637ec..a5077a6f4d9f 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java @@ -14,6 +14,8 @@ import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.model.generated.DiscoverCatalogResult; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaWriteRequestBody; +import io.airbyte.commons.converters.CatalogClientConverters; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConnectorJobOutput; @@ -28,8 +30,6 @@ import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.helper.CatalogClientConverters; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.DefaultAirbyteStreamFactory; import io.airbyte.workers.process.IntegrationLauncher; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index bae558f504a8..60378fd83ca2 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -14,6 +14,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import datadog.trace.api.Trace; +import io.airbyte.commons.converters.ConnectorConfigUpdater; +import io.airbyte.commons.converters.ThreadedTimeTracker; import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.FailureReason; import io.airbyte.config.ReplicationAttemptSummary; @@ -37,9 +39,7 @@ import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.RecordSchemaValidationException; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.helper.FailureHelper; -import io.airbyte.workers.helper.ThreadedTimeTracker; import io.airbyte.workers.internal.AirbyteDestination; import io.airbyte.workers.internal.AirbyteMapper; import io.airbyte.workers.internal.AirbyteSource; @@ -606,8 +606,8 @@ private List getFailureReasons(final AtomicReference> streamToAllFields, - Map> unexpectedFields, + final Map> streamToAllFields, + final Map> unexpectedFields, final Map, Integer>> validationErrors, final AirbyteMessage message) { if (message.getRecord() == null) { @@ -639,10 +639,12 @@ private static void validateSchema(final RecordSchemaValidator recordSchemaValid } } - private static void populateUnexpectedFieldNames(AirbyteRecordMessage record, Set fieldsInCatalog, Set unexpectedFieldNames) { + private static void populateUnexpectedFieldNames(final AirbyteRecordMessage record, + final Set fieldsInCatalog, + final Set unexpectedFieldNames) { final JsonNode data = record.getData(); if (data.isObject()) { - Iterator fieldNamesInRecord = data.fieldNames(); + final Iterator fieldNamesInRecord = data.fieldNames(); while (fieldNamesInRecord.hasNext()) { final String fieldName = fieldNamesInRecord.next(); if (!fieldsInCatalog.contains(fieldName)) { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java index b263a494e02c..9c48060b9f04 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -53,7 +54,6 @@ import io.airbyte.workers.WorkerMetricReporter; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.helper.FailureHelper; import io.airbyte.workers.internal.AirbyteDestination; import io.airbyte.workers.internal.AirbyteSource; diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerPerformanceTest.java index 641023cc88b2..e7fdede3159b 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerPerformanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.workers.general; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.protocol.AirbyteMessageMigrator; import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; @@ -26,7 +27,6 @@ import io.airbyte.workers.RecordSchemaValidator; import io.airbyte.workers.WorkerMetricReporter; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.DefaultAirbyteSource; import io.airbyte.workers.internal.NamespacingMapper; import io.airbyte.workers.internal.VersionedAirbyteStreamFactory; diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4217e59dc282..bdc9c5f3b71b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -728,7 +728,7 @@ - name: Google Sheets sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 dockerRepository: airbyte/source-google-sheets - dockerImageTag: 0.2.31 + dockerImageTag: 0.2.32 documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets icon: google-sheets.svg sourceType: file @@ -1376,7 +1376,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.44 + dockerImageTag: 1.0.45 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 965df6ef08ba..a1b408582eb4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6044,7 +6044,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-sheets:0.2.31" +- dockerImage: "airbyte/source-google-sheets:0.2.32" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-sheets" connectionSpecification: @@ -11617,7 +11617,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.44" +- dockerImage: "airbyte/source-postgres:1.0.45" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-container-orchestrator/build.gradle b/airbyte-container-orchestrator/build.gradle index 835e34290650..dd6d443b16cb 100644 --- a/airbyte-container-orchestrator/build.gradle +++ b/airbyte-container-orchestrator/build.gradle @@ -30,6 +30,7 @@ dependencies { implementation project(':airbyte-api') implementation project(':airbyte-config:config-models') + implementation project(':airbyte-commons-converters') implementation project(':airbyte-commons-protocol') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-commons-worker') diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java index fadbeaad5207..24b9cb21f98c 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java @@ -12,6 +12,7 @@ import datadog.trace.api.Trace; import io.airbyte.api.client.generated.DestinationApi; import io.airbyte.api.client.generated.SourceApi; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.FeatureFlagHelper; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.json.Jsons; @@ -37,7 +38,6 @@ import io.airbyte.workers.WorkerMetricReporter; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.general.DefaultReplicationWorker; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.DefaultAirbyteDestination; import io.airbyte.workers.internal.DefaultAirbyteSource; diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index d66d00adc0c2..e53f17bfdca3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.6.0 +Allow passing custom environment variables to the connector under test. [#22937](https://github.com/airbytehq/airbyte/pull/22937). + ## 0.5.3 Spec tests: Make `oneOf` checks work for nested `oneOf`s. [#22395](https://github.com/airbytehq/airbyte/pull/22395) diff --git a/airbyte-integrations/bases/connector-acceptance-test/Dockerfile b/airbyte-integrations/bases/connector-acceptance-test/Dockerfile index 6c35d67a2942..9d345b9165a1 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/connector-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY connector_acceptance_test ./connector_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.5.3 +LABEL io.airbyte.version=0.6.0 LABEL io.airbyte.name=airbyte/connector-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index 283e436cced6..45fa61605c87 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -200,6 +200,9 @@ class TestStrictnessLevel(str, Enum): default=TestStrictnessLevel.low, description="Corresponds to a strictness level of the test suite and will change which tests are mandatory for a successful run.", ) + custom_environment_variables: Optional[Mapping] = Field( + default={}, description="Mapping of custom environment variables to pass to the connector under test." + ) @staticmethod def is_legacy(config: dict) -> bool: diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index 453e918fd1e6..ab187ece9231 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -11,7 +11,7 @@ from logging import Logger from pathlib import Path from subprocess import STDOUT, check_output, run -from typing import Any, List, MutableMapping, Optional, Set +from typing import Any, List, Mapping, MutableMapping, Optional, Set import pytest from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConnectorSpecification, Type @@ -55,6 +55,11 @@ def cache_discovered_catalog_fixture(acceptance_test_config: Config) -> bool: return acceptance_test_config.cache_discovered_catalog +@pytest.fixture(name="custom_environment_variables", scope="session") +def custom_environment_variables_fixture(acceptance_test_config: Config) -> Mapping: + return acceptance_test_config.custom_environment_variables + + @pytest.fixture(name="connector_config_path") def connector_config_path_fixture(inputs, base_path) -> Path: """Fixture with connector's config path. The path to the latest updated configurations will be returned if any.""" @@ -140,8 +145,13 @@ def connector_spec_fixture(connector_spec_path) -> ConnectorSpecification: @pytest.fixture(name="docker_runner") -def docker_runner_fixture(image_tag, tmp_path, connector_config_path) -> ConnectorRunner: - return ConnectorRunner(image_tag, volume=tmp_path, connector_configuration_path=connector_config_path) +def docker_runner_fixture(image_tag, tmp_path, connector_config_path, custom_environment_variables) -> ConnectorRunner: + return ConnectorRunner( + image_tag, + volume=tmp_path, + connector_configuration_path=connector_config_path, + custom_environment_variables=custom_environment_variables, + ) @pytest.fixture(name="previous_connector_image_name") @@ -308,7 +318,7 @@ def detailed_logger() -> Logger: logger.setLevel(logging.DEBUG) fh = logging.FileHandler(filename, mode="w") fh.setFormatter(formatter) - logger.log_json_list = lambda l: logger.info(json.dumps(list(l), indent=1)) + logger.log_json_list = lambda line: logger.info(json.dumps(list(line), indent=1)) logger.handlers = [fh] return logger diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py index 0a748ea75277..b22d2d5bfcfc 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py @@ -17,7 +17,13 @@ class ConnectorRunner: - def __init__(self, image_name: str, volume: Path, connector_configuration_path: Optional[Path] = None): + def __init__( + self, + image_name: str, + volume: Path, + connector_configuration_path: Optional[Path] = None, + custom_environment_variables: Optional[Mapping] = {}, + ): self._client = docker.from_env() try: self._image = self._client.images.get(image_name) @@ -28,6 +34,7 @@ def __init__(self, image_name: str, volume: Path, connector_configuration_path: self._runs = 0 self._volume_base = volume self._connector_configuration_path = connector_configuration_path + self._custom_environment_variables = custom_environment_variables @property def output_folder(self) -> Path: @@ -106,6 +113,7 @@ def run(self, cmd, config=None, state=None, catalog=None, raise_container_error: volumes=volumes, network_mode="host", detach=True, + environment=self._custom_environment_variables, **kwargs, ) with open(self.output_folder / "raw", "wb+") as f: diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_container_runner.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_container_runner.py index d658d9571913..d1716de8f76e 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_container_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_container_runner.py @@ -37,10 +37,19 @@ def test_run_call_persist_configuration(self, mocker, tmp_path): ] mocker.patch.object(connector_runner.ConnectorRunner, "read", mocker.Mock(return_value=records_reads)) mocker.patch.object(connector_runner.ConnectorRunner, "_persist_new_configuration") - - runner = connector_runner.ConnectorRunner("source-test:dev", tmp_path, connector_configuration_path=old_configuration_path) + runner = connector_runner.ConnectorRunner( + "source-test:dev", tmp_path, connector_configuration_path=old_configuration_path, custom_environment_variables={"foo": "bar"} + ) list(runner.run("dummy_cmd")) runner._persist_new_configuration.assert_called_once_with(new_configuration, 1) + runner._client.containers.run.assert_called_once_with( + image=runner._image, + command="dummy_cmd", + volumes={str(tmp_path) + "/run_1/input": {"bind": "/data"}, str(tmp_path) + "/run_1/output": {"bind": "/local", "mode": "rw"}}, + network_mode="host", + detach=True, + environment={"foo": "bar"}, + ) @pytest.mark.parametrize( "pass_configuration_path, old_configuration, new_configuration, new_configuration_emitted_at, expect_new_configuration", diff --git a/airbyte-integrations/bases/standard-destination-test/build.gradle b/airbyte-integrations/bases/standard-destination-test/build.gradle index a6ef45eb9abc..763694ca6e0a 100644 --- a/airbyte-integrations/bases/standard-destination-test/build.gradle +++ b/airbyte-integrations/bases/standard-destination-test/build.gradle @@ -3,6 +3,7 @@ plugins { } dependencies { implementation project(':airbyte-db:db-lib') + implementation project(':airbyte-commons-converters') implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:init') diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 92a30951551a..3d8269c80456 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; @@ -58,7 +59,6 @@ import io.airbyte.workers.general.DbtTransformationRunner; import io.airbyte.workers.general.DefaultCheckConnectionWorker; import io.airbyte.workers.general.DefaultGetSpecWorker; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.helper.EntrypointEnvChecker; import io.airbyte.workers.internal.AirbyteDestination; import io.airbyte.workers.internal.DefaultAirbyteDestination; diff --git a/airbyte-integrations/bases/standard-source-test/build.gradle b/airbyte-integrations/bases/standard-source-test/build.gradle index 9d64af20a834..4e048a42c0f1 100644 --- a/airbyte-integrations/bases/standard-source-test/build.gradle +++ b/airbyte-integrations/bases/standard-source-test/build.gradle @@ -14,6 +14,7 @@ import org.jsoup.Jsoup; dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-api') + implementation project(':airbyte-commons-converters') implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java index e245ec5df6df..575bafe62fe4 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java @@ -16,6 +16,8 @@ import io.airbyte.api.client.generated.SourceApi; import io.airbyte.api.client.model.generated.DiscoverCatalogResult; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaWriteRequestBody; +import io.airbyte.commons.converters.CatalogClientConverters; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.config.EnvConfigs; @@ -37,8 +39,6 @@ import io.airbyte.workers.general.DefaultCheckConnectionWorker; import io.airbyte.workers.general.DefaultDiscoverCatalogWorker; import io.airbyte.workers.general.DefaultGetSpecWorker; -import io.airbyte.workers.helper.CatalogClientConverters; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.helper.EntrypointEnvChecker; import io.airbyte.workers.internal.AirbyteSource; import io.airbyte.workers.internal.DefaultAirbyteSource; diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index 31af71d2aa2a..dab88d6d9cd8 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -239,13 +239,13 @@ def get_updated_state( { "stream": { - "http://domain1.com": { + "https://domain1.com": { "web": {"date": "2022-01-03"}, "news": {"date": "2022-01-03"}, "image": {"date": "2022-01-03"}, "video": {"date": "2022-01-03"} }, - "http://domain2.com": { + "https://domain2.com": { "web": {"date": "2022-01-03"}, "news": {"date": "2022-01-03"}, "image": {"date": "2022-01-03"}, diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 469e33a5964a..68b751f06b45 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.31 +LABEL io.airbyte.version=0.2.32 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py index ef948e0a4c34..96b692310c82 100644 --- a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/helpers.py @@ -199,7 +199,7 @@ def row_contains_relevant_data(cell_values: List[str], relevant_indices: Iterabl @staticmethod def get_spreadsheet_id(id_or_url: str) -> str: - if re.match(r"(http://)|(https://)", id_or_url): + if re.match(r"(https://)", id_or_url): # This is a URL m = re.search(r"(/)([-\w]{40,})([/]?)", id_or_url) if m is not None and m.group(2): diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py index cca5e1475b10..2d362f23b5fb 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py @@ -236,15 +236,11 @@ def test_get_spreadsheet_id(self): result = Helpers.get_spreadsheet_id(test_url) self.assertEqual("18vWlVH8BfjGa-gwYGdV1BjcPP9re66xI8uJK25dtY9Q", result) - test_url = "http://docs.google.com/spreadsheets/d/18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q/" + test_url = "https://docs.google.com/spreadsheets/d/18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q/" result = Helpers.get_spreadsheet_id(test_url) self.assertEqual("18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q", result) - test_url = "http://docs.google.com/spreadsheets/d/18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q/#" - result = Helpers.get_spreadsheet_id(test_url) - self.assertEqual("18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q", result) - - test_url = "http://docs.google.com/spreadsheets/d/18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q" + test_url = "https://docs.google.com/spreadsheets/d/18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q/#" result = Helpers.get_spreadsheet_id(test_url) self.assertEqual("18vWlVH8BfjGegwY_GdV1BjcPP9re_6xI8uJ-25dtY9Q", result) diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 144619c15cfe..e7cd058907bb 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.44 +LABEL io.airbyte.version=1.0.45 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 4ef9e47b37e9..0e3f5772d059 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.44 +LABEL io.airbyte.version=1.0.45 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 18d9d202a275..770e2bfac814 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -2,6 +2,8 @@ # for more information about how to configure these tests connector_image: airbyte/source-postgres:dev test_strictness_level: high +custom_environment_variables: + USE_STREAM_CAPABLE_STATE: true acceptance_tests: spec: tests: diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index f98df703c94c..1197aafcb3b9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedConsumer; @@ -92,6 +93,8 @@ public class PostgresSource extends AbstractJdbcSource implements private static final Logger LOGGER = LoggerFactory.getLogger(PostgresSource.class); private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; + public static final String REPLICATION_PRIVILEGE_ERROR_MESSAGE = + "User '%s' does not have enough privileges for CDC replication. Please read the docs and add required privileges."; public static final String PARAM_SSLMODE = "sslmode"; public static final String SSL_MODE = "ssl_mode"; public static final String PARAM_SSL = "ssl"; @@ -288,7 +291,7 @@ public List> getCheckOperations(final J final List matchingSlots = getReplicationSlot(database, config); if (matchingSlots.size() != 1) { - throw new RuntimeException( + throw new ConfigErrorException( "Expected exactly one replication slot but found " + matchingSlots.size() + ". Please read the docs and add a replication slot to your database."); } @@ -304,7 +307,7 @@ public List> getCheckOperations(final J }, sourceOperations::rowToJson); if (matchingPublications.size() != 1) { - throw new RuntimeException( + throw new ConfigErrorException( "Expected exactly one publication but found " + matchingPublications.size() + ". Please read the docs and add a publication to your database."); } @@ -314,6 +317,23 @@ public List> getCheckOperations(final J checkOperations.add(database -> { PostgresUtils.checkFirstRecordWaitTime(config); }); + + // Verify that the db user has required privilege to perform replication. + checkOperations.add(database -> { + final String userName = config.get("username").asText(); + + final List userPrivileges = database.queryJsons(connection -> { + final PreparedStatement ps = connection.prepareStatement("SELECT userepl FROM pg_user WHERE usename = ?"); + ps.setString(1, userName); + LOGGER.info("Verifying required privileges for user: {}", userName); + return ps; + }, sourceOperations::rowToJson); + + if (!userPrivileges.get(0).get("userepl").asBoolean()) { + throw new ConfigErrorException(String.format(REPLICATION_PRIVILEGE_ERROR_MESSAGE, userName)); + } + + }); } return checkOperations; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 3e8de04423c4..b45b9821aaf3 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -36,6 +36,7 @@ import io.airbyte.integrations.base.Source; import io.airbyte.integrations.debezium.CdcSourceTest; import io.airbyte.integrations.debezium.CdcTargetPosition; +import io.airbyte.integrations.util.ConnectorExceptionUtil; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -166,6 +167,20 @@ private static DSLContext getDslContext(final JsonNode config) { SQLDialect.POSTGRES); } + private void revokeReplicationPermission() { + executeQuery("ALTER USER " + container.getUsername() + " NOREPLICATION;"); + } + + @Test + void testCheckWithoutReplicationPermission() throws Exception { + revokeReplicationPermission(); + final AirbyteConnectionStatus status = source.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertEquals(String.format(ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE, + String.format(PostgresSource.REPLICATION_PRIVILEGE_ERROR_MESSAGE, config.get("username").asText())), + status.getMessage()); + } + @Test void testCheckWithoutPublication() throws Exception { database.query(ctx -> ctx.execute("DROP PUBLICATION " + PUBLICATION + ";")); diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java index f6df27ada8e6..2f092d48a67b 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java @@ -31,6 +31,7 @@ import io.airbyte.integrations.source.relationaldb.models.DbState; import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; +import io.airbyte.integrations.util.ConnectorExceptionUtil; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; @@ -98,7 +99,7 @@ public AirbyteConnectionStatus check(final JsonNode config) throws Exception { LOGGER.info("Exception while checking connection: ", e); return new AirbyteConnectionStatus() .withStatus(Status.FAILED) - .withMessage("Could not connect with provided configuration. Error: " + e.getMessage()); + .withMessage(String.format(ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE, e.getMessage())); } finally { close(); } diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 88142f575412..0d787f67a144 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.0.2 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index 39892e69be8c..bab87b176a4d 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -19,10 +19,11 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" - empty_streams: ["external_account_bank_accounts"] + empty_streams: ["bank_accounts", "checkout_sessions", "checkout_sessions_line_items", "external_account_bank_accounts"] # TEST 1 - Reading catalog without invoice_line_items - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" + empty_streams: ["transfers"] timeout_seconds: 3600 # TEST 2 - Reading data from account that has no records for stream Disputes - config_path: "secrets/connected_account_config.json" diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py new file mode 100644 index 000000000000..37734f1cf0be --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py @@ -0,0 +1,24 @@ +import logging +from typing import Optional, Tuple + +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy + + +class StripeSubStreamAvailabilityStrategy(HttpAvailabilityStrategy): + def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional[Source]) -> Tuple[bool, Optional[str]]: + """Traverse through all the parents of a given stream and run availability strategy on each of them""" + try: + current_stream, parent_stream = stream, getattr(stream, "parent") + except AttributeError: + return super().check_availability(stream, logger, source) + if parent_stream: + parent_stream_instance = getattr(current_stream, "get_parent_stream_instance")() + # Accessing the `availability_strategy` property will instantiate AvailabilityStrategy under the hood + availability_strategy = parent_stream_instance.availability_strategy + if availability_strategy: + is_available, reason = availability_strategy.check_availability(parent_stream_instance, logger, source) + if not is_available: + return is_available, reason + return super().check_availability(stream, logger, source) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index c8cc0217366f..044a8b69f8b5 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -13,6 +13,8 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream +from source_stripe.availability_strategy import StripeSubStreamAvailabilityStrategy + STRIPE_ERROR_CODES: List = [ # stream requires additional permissions "more_permissions_required", @@ -32,10 +34,6 @@ def __init__(self, start_date: int, account_id: str, slice_range: int = DEFAULT_ self.start_date = start_date self.slice_range = slice_range or self.DEFAULT_SLICE_RANGE - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() if bool(decoded_response.get("has_more", "False")) and decoded_response.get("data", []): @@ -318,6 +316,10 @@ def sub_items_attr(self) -> str: If the stream has no primary keys, return None. """ + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): params = super().request_params(stream_slice=stream_slice, **kwargs) @@ -327,8 +329,11 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): return params + def get_parent_stream_instance(self): + return self.parent(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) + def read_records(self, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - parent_stream = self.parent(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) + parent_stream = self.get_parent_stream_instance() slices = parent_stream.stream_slices(sync_mode=SyncMode.full_refresh) for _slice in slices: for record in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py new file mode 100644 index 000000000000..aaa83c1810e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py @@ -0,0 +1,119 @@ +import pendulum +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy + +from source_stripe.availability_strategy import StripeSubStreamAvailabilityStrategy +from source_stripe.streams import InvoiceLineItems, Invoices + + +def test_traverse_over_substreams(mocker): + # Mock base HttpAvailabilityStrategy to capture all the check_availability method calls + check_availability_mock = mocker.MagicMock() + check_availability_mock.return_value = (True, None) + mocker.patch( + "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", + check_availability_mock + ) + + # Prepare tree of nested objects + root = mocker.Mock() + root.availability_strategy = HttpAvailabilityStrategy() + root.parent = None + + child_1 = mocker.Mock() + child_1.availability_strategy = StripeSubStreamAvailabilityStrategy() + child_1.get_parent_stream_instance.return_value = root + + child_1_1 = mocker.Mock() + child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() + child_1_1.get_parent_stream_instance.return_value = child_1 + + child_1_1_1 = mocker.Mock() + child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() + child_1_1_1.get_parent_stream_instance.return_value = child_1_1 + + # Start traverse + is_available, reason = child_1_1_1.availability_strategy.check_availability(child_1_1_1, mocker.Mock(), mocker.Mock()) + + assert is_available and reason is None + + # Check availability strategy was called once for every nested object + assert check_availability_mock.call_count == 4 + + # Check each availability strategy was called with proper instance argument + assert id(check_availability_mock.call_args_list[0].args[0]) == id(root) + assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1) + assert id(check_availability_mock.call_args_list[2].args[0]) == id(child_1_1) + assert id(check_availability_mock.call_args_list[3].args[0]) == id(child_1_1_1) + + +def test_traverse_over_substreams_failure(mocker): + # Mock base HttpAvailabilityStrategy to capture all the check_availability method calls + check_availability_mock = mocker.MagicMock() + check_availability_mock.side_effect = [(True, None), (False, "child_1")] + mocker.patch( + "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", + check_availability_mock + ) + + # Prepare tree of nested objects + root = mocker.Mock() + root.availability_strategy = HttpAvailabilityStrategy() + root.parent = None + + child_1 = mocker.Mock() + child_1.availability_strategy = StripeSubStreamAvailabilityStrategy() + child_1.get_parent_stream_instance.return_value = root + + child_1_1 = mocker.Mock() + child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() + child_1_1.get_parent_stream_instance.return_value = child_1 + + child_1_1_1 = mocker.Mock() + child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() + child_1_1_1.get_parent_stream_instance.return_value = child_1_1 + + # Start traverse + is_available, reason = child_1_1_1.availability_strategy.check_availability(child_1_1_1, mocker.Mock(), mocker.Mock()) + + assert not is_available and reason == "child_1" + + # Check availability strategy was called once for every nested object + assert check_availability_mock.call_count == 2 + + # Check each availability strategy was called with proper instance argument + assert id(check_availability_mock.call_args_list[0].args[0]) == id(root) + assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1) + + +def test_substream_availability(mocker): + check_availability_mock = mocker.MagicMock() + check_availability_mock.return_value = (True, None) + mocker.patch( + "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", + check_availability_mock + ) + + stream = InvoiceLineItems(start_date=pendulum.today().subtract(days=3).int_timestamp, account_id="None") + is_available, reason = stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock()) + assert is_available and reason is None + + assert check_availability_mock.call_count == 2 + assert isinstance(check_availability_mock.call_args_list[0].args[0], Invoices) + assert isinstance(check_availability_mock.call_args_list[1].args[0], InvoiceLineItems) + + +def test_substream_availability_no_parent(mocker): + check_availability_mock = mocker.MagicMock() + check_availability_mock.return_value = (True, None) + mocker.patch( + "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", + check_availability_mock + ) + + stream = InvoiceLineItems(start_date=pendulum.today().subtract(days=3).int_timestamp, account_id="None") + stream.parent = None + + stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock()) + + assert check_availability_mock.call_count == 1 + assert isinstance(check_availability_mock.call_args_list[0].args[0], InvoiceLineItems) diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java index 8c5575adcabc..59e540a4f86d 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java @@ -14,6 +14,11 @@ public final class ApmTraceConstants { */ public static final String ACTIVITY_TRACE_OPERATION_NAME = "activity"; + /** + * Operation name for an APM trace of API endpoint execution. + */ + public static final String ENDPOINT_EXECUTION_OPERATION_NAME = "execute"; + /** * Operation name for an APM trace of a job orchestrator. */ diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java index 95a624770a87..6175ae30da82 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java @@ -137,7 +137,8 @@ public Optional createResetConnectionJob(final DestinationConnection desti workerResourceRequirements)) .withResetSourceConfiguration(new ResetSourceConfiguration().withStreamsToReset(streamsToReset)) .withIsSourceCustomConnector(false) - .withIsDestinationCustomConnector(isDestinationCustomConnector); + .withIsDestinationCustomConnector(isDestinationCustomConnector) + .withWorkspaceId(destination.getWorkspaceId()); final JobConfig jobConfig = new JobConfig() .withConfigType(ConfigType.RESET_CONNECTION) diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java index d301b60e3e51..77c8e35fd265 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java @@ -412,7 +412,8 @@ void testCreateResetConnectionJob() throws IOException { .withResourceRequirements(workerResourceRequirements) .withResetSourceConfiguration(new ResetSourceConfiguration().withStreamsToReset(streamsToReset)) .withIsSourceCustomConnector(false) - .withIsDestinationCustomConnector(false); + .withIsDestinationCustomConnector(false) + .withWorkspaceId(DESTINATION_CONNECTION.getWorkspaceId()); final JobConfig jobConfig = new JobConfig() .withConfigType(ConfigType.RESET_CONNECTION) @@ -464,7 +465,8 @@ void testCreateResetConnectionJobEnsureNoQueuing() throws IOException { .withResourceRequirements(workerResourceRequirements) .withResetSourceConfiguration(new ResetSourceConfiguration().withStreamsToReset(streamsToReset)) .withIsSourceCustomConnector(false) - .withIsDestinationCustomConnector(false); + .withIsDestinationCustomConnector(false) + .withWorkspaceId(DESTINATION_CONNECTION.getWorkspaceId()); final JobConfig jobConfig = new JobConfig() .withConfigType(ConfigType.RESET_CONNECTION) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index d88d9e30f5ff..19ae4999307e 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -33,9 +33,9 @@ dependencies { implementation project(':airbyte-analytics') implementation project(':airbyte-api') + implementation project(':airbyte-commons-converters') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-commons-server') - implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:init') implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') @@ -51,6 +51,7 @@ dependencies { implementation 'com.github.slugify:slugify:2.4' implementation 'commons-cli:commons-cli:1.4' implementation libs.temporal.sdk + implementation libs.bundles.datadog implementation 'org.apache.cxf:cxf-core:3.4.2' implementation 'org.eclipse.jetty:jetty-server:9.4.31.v20200723' implementation 'org.eclipse.jetty:jetty-servlet:9.4.31.v20200723' diff --git a/airbyte-server/src/main/java/io/airbyte/server/LoggingEventListener.java b/airbyte-server/src/main/java/io/airbyte/server/LoggingEventListener.java new file mode 100644 index 000000000000..2bf3239b9335 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/LoggingEventListener.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server; + +import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.helpers.LogClientSingleton; +import io.airbyte.config.helpers.LogConfigs; +import io.micronaut.context.annotation.Value; +import io.micronaut.context.event.ApplicationEventListener; +import io.micronaut.discovery.event.ServiceReadyEvent; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import java.nio.file.Path; +import java.util.Optional; + +/** + * Initializes the logging client on startup + */ +@Singleton +public class LoggingEventListener implements ApplicationEventListener { + + @Inject + private Optional logConfigs; + @Inject + private WorkerEnvironment workerEnvironment; + @Value("${airbyte.workspace.root}") + private String workspaceRoot; + + @Override + public void onApplicationEvent(final ServiceReadyEvent event) { + // Configure logging client + LogClientSingleton.getInstance().setWorkspaceMdc(workerEnvironment, logConfigs.orElseThrow(), + LogClientSingleton.getInstance().getServerLogsRoot(Path.of(workspaceRoot))); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 6cb04bf40dc9..5390e251b467 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -7,12 +7,35 @@ import io.airbyte.analytics.Deployment; import io.airbyte.analytics.TrackingClient; import io.airbyte.analytics.TrackingClientSingleton; +import io.airbyte.commons.converters.ConnectionHelper; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.server.RequestLogger; -import io.airbyte.commons.server.errors.*; -import io.airbyte.commons.server.handlers.*; +import io.airbyte.commons.server.errors.InvalidInputExceptionMapper; +import io.airbyte.commons.server.errors.InvalidJsonExceptionMapper; +import io.airbyte.commons.server.errors.InvalidJsonInputExceptionMapper; +import io.airbyte.commons.server.errors.KnownExceptionMapper; +import io.airbyte.commons.server.errors.NotFoundExceptionMapper; +import io.airbyte.commons.server.errors.UncaughtExceptionMapper; +import io.airbyte.commons.server.handlers.AttemptHandler; +import io.airbyte.commons.server.handlers.ConnectionsHandler; +import io.airbyte.commons.server.handlers.DestinationDefinitionsHandler; +import io.airbyte.commons.server.handlers.DestinationHandler; +import io.airbyte.commons.server.handlers.HealthCheckHandler; +import io.airbyte.commons.server.handlers.JobHistoryHandler; +import io.airbyte.commons.server.handlers.LogsHandler; +import io.airbyte.commons.server.handlers.OAuthHandler; +import io.airbyte.commons.server.handlers.OpenApiConfigHandler; +import io.airbyte.commons.server.handlers.OperationsHandler; +import io.airbyte.commons.server.handlers.SchedulerHandler; +import io.airbyte.commons.server.handlers.SourceDefinitionsHandler; +import io.airbyte.commons.server.handlers.SourceHandler; +import io.airbyte.commons.server.handlers.StateHandler; +import io.airbyte.commons.server.handlers.WebBackendCheckUpdatesHandler; +import io.airbyte.commons.server.handlers.WebBackendConnectionsHandler; +import io.airbyte.commons.server.handlers.WebBackendGeographiesHandler; +import io.airbyte.commons.server.handlers.WorkspacesHandler; import io.airbyte.commons.server.scheduler.DefaultSynchronousSchedulerClient; import io.airbyte.commons.server.scheduler.EventRunner; import io.airbyte.commons.server.scheduler.TemporalEventRunner; @@ -49,7 +72,6 @@ import io.airbyte.persistence.job.factory.OAuthConfigSupplier; import io.airbyte.persistence.job.tracker.JobTracker; import io.airbyte.validation.json.JsonSchemaValidator; -import io.airbyte.workers.helper.ConnectionHelper; import io.temporal.client.WorkflowClient; import io.temporal.serviceclient.WorkflowServiceStubs; import java.net.http.HttpClient; @@ -229,7 +251,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, streamResetRecordsHelper); final OAuthConfigSupplier oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, trackingClient); - RouterService routerService = new RouterService(configRepository, taskQueueMapper); + final RouterService routerService = new RouterService(configRepository, taskQueueMapper); final DefaultSynchronousSchedulerClient syncSchedulerClient = new DefaultSynchronousSchedulerClient(temporalClient, jobTracker, jobErrorReporter, oAuthConfigSupplier, routerService); final HttpClient httpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java index de4cdbd3c30f..066cc1ddc8cb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java @@ -4,27 +4,36 @@ package io.airbyte.server.apis; +import static io.airbyte.metrics.lib.ApmTraceConstants.ENDPOINT_EXECUTION_OPERATION_NAME; + +import datadog.trace.api.Trace; import io.airbyte.commons.server.errors.BadObjectSchemaKnownException; import io.airbyte.commons.server.errors.IdNotFoundKnownException; import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import org.slf4j.LoggerFactory; public class ApiHelper { + @Trace(operationName = ENDPOINT_EXECUTION_OPERATION_NAME) static T execute(final HandlerCall call) { try { return call.call(); } catch (final ConfigNotFoundException e) { + ApmTraceUtils.recordErrorOnRootSpan(e); throw new IdNotFoundKnownException(String.format("Could not find configuration for %s: %s.", e.getType(), e.getConfigId()), e.getConfigId(), e); } catch (final JsonValidationException e) { + ApmTraceUtils.recordErrorOnRootSpan(e); throw new BadObjectSchemaKnownException( String.format("The provided configuration does not fulfill the specification. Errors: %s", e.getMessage()), e); } catch (final IOException e) { + ApmTraceUtils.recordErrorOnRootSpan(e); throw new RuntimeException(e); } catch (final Exception e) { + ApmTraceUtils.recordErrorOnRootSpan(e); LoggerFactory.getLogger(ApiHelper.class).error("Unexpected Exception", e); throw e; } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java index 9c6949b9b36e..e3c8168f589c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java @@ -12,7 +12,6 @@ import io.airbyte.api.model.generated.SaveStatsRequestBody; import io.airbyte.api.model.generated.SetWorkflowInAttemptRequestBody; import io.airbyte.commons.server.handlers.AttemptHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; @@ -20,9 +19,7 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; -@Controller("/api/v1/attempt/") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") +@Controller("/api/v1/attempt") @Secured(SecurityRule.IS_AUTHENTICATED) public class AttemptApiController implements AttemptApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index 5a86ff1dfa54..d58f06923f0b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -21,7 +21,6 @@ import io.airbyte.commons.server.handlers.OperationsHandler; import io.airbyte.commons.server.handlers.SchedulerHandler; import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; @@ -31,9 +30,7 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/connections") -@Context() -@Requires(property = "airbyte.deployment-mode", - value = "OSS") +@Context @Secured(SecurityRule.IS_AUTHENTICATED) public class ConnectionApiController implements ConnectionApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java index d594124f998c..e8144d6a4510 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java @@ -20,7 +20,6 @@ import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.DestinationHandler; import io.airbyte.commons.server.handlers.SchedulerHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; @@ -30,8 +29,6 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destinations") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationApiController implements DestinationApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java index 2f86a587082f..0654b02aaf65 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java @@ -22,7 +22,6 @@ import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.DestinationDefinitionsHandler; import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; @@ -31,8 +30,6 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destination_definitions") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Context @Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationDefinitionApiController implements DestinationDefinitionApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java index 92dc7cae20e8..3dbe1ddeffcb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java @@ -10,15 +10,12 @@ import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; import io.airbyte.commons.server.handlers.SchedulerHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/destination_definition_specifications") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationDefinitionSpecificationApiController implements DestinationDefinitionSpecificationApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java index 7b747dafa47f..0c23e972a898 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationOauthApiController.java @@ -15,7 +15,6 @@ import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.OAuthHandler; import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; @@ -23,8 +22,6 @@ import java.util.Map; @Controller("/api/v1/destination_oauths") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Context @Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationOauthApiController implements DestinationOauthApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java index 0e8622c52ddd..8df941f52495 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java @@ -7,7 +7,6 @@ import io.airbyte.api.generated.HealthApi; import io.airbyte.api.model.generated.HealthCheckRead; import io.airbyte.commons.server.handlers.HealthCheckHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; @@ -15,8 +14,6 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/health") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_ANONYMOUS) public class HealthApiController implements HealthApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java index 4fc06076ab28..da6128a78a21 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java @@ -22,15 +22,12 @@ import io.airbyte.commons.server.handlers.JobHistoryHandler; import io.airbyte.commons.server.handlers.SchedulerHandler; import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/jobs") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Context @Secured(SecurityRule.IS_AUTHENTICATED) public class JobsApiController implements JobsApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java index 97e1050fc518..ef1844e0bc7a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java @@ -10,7 +10,6 @@ import io.airbyte.api.model.generated.LogsRequestBody; import io.airbyte.commons.server.handlers.LogsHandler; import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; @@ -18,8 +17,6 @@ import java.io.File; @Controller("/api/v1/logs") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Context @Secured(SecurityRule.IS_AUTHENTICATED) public class LogsApiController implements LogsApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java index d278cde04c5d..b9dda0e7f11c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java @@ -4,7 +4,6 @@ package io.airbyte.server.apis; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; @@ -18,8 +17,6 @@ * Custom controller that handles global 404 responses for unknown/unmapped paths. */ @Controller("/api/notfound") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_ANONYMOUS) public class NotFoundController { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java index cf8c0a57180d..5cdf8e9c11e5 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java @@ -13,7 +13,6 @@ import io.airbyte.commons.server.converters.NotificationConverter; import io.airbyte.commons.server.errors.IdNotFoundKnownException; import io.airbyte.notification.NotificationClient; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; @@ -21,15 +20,13 @@ import io.micronaut.security.rules.SecurityRule; import java.io.IOException; -@Controller("/api/v1/notifications/try") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") +@Controller("/api/v1/notifications") @Secured(SecurityRule.IS_AUTHENTICATED) public class NotificationsApiController implements NotificationsApi { public NotificationsApiController() {} - @Post + @Post("/try") @Secured({AUTHENTICATED_USER}) @Override public NotificationRead tryNotificationConfig(@Body final Notification notification) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java index ee5b4b6d0be8..0858a6092492 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/OpenapiApiController.java @@ -8,7 +8,6 @@ import io.airbyte.api.generated.OpenapiApi; import io.airbyte.commons.server.handlers.OpenApiConfigHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; import io.micronaut.security.annotation.Secured; @@ -16,8 +15,6 @@ import java.io.File; @Controller("/api/v1/openapi") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class OpenapiApiController implements OpenapiApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java index 8065ea9280b6..85d38e107d68 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/OperationApiController.java @@ -19,7 +19,6 @@ import io.airbyte.api.model.generated.OperatorConfiguration; import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.OperationsHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; @@ -29,8 +28,6 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/operations") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class OperationApiController implements OperationApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java index d7a5edf6f652..d2c4aaac77cb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SchedulerApiController.java @@ -14,15 +14,12 @@ import io.airbyte.api.model.generated.SourceDiscoverSchemaRead; import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.SchedulerHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/scheduler") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class SchedulerApiController implements SchedulerApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java index 4ad1f408ff05..e3a36a79f354 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java @@ -25,7 +25,6 @@ import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.SchedulerHandler; import io.airbyte.commons.server.handlers.SourceHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; @@ -34,8 +33,6 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/sources") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class SourceApiController implements SourceApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java index f7d3a233b03d..a390b63bc128 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionApiController.java @@ -22,7 +22,6 @@ import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.SourceDefinitionsHandler; import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; @@ -31,8 +30,6 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/source_definitions") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Context @Secured(SecurityRule.IS_AUTHENTICATED) public class SourceDefinitionApiController implements SourceDefinitionApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java index 0bf03a8fd403..3711795a09f2 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java @@ -10,15 +10,12 @@ import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; import io.airbyte.commons.server.handlers.SchedulerHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/source_definition_specifications") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class SourceDefinitionSpecificationApiController implements SourceDefinitionSpecificationApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java index 14f0069e260a..3857eb86108f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceOauthApiController.java @@ -14,7 +14,6 @@ import io.airbyte.api.model.generated.SourceOauthConsentRequest; import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.OAuthHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; @@ -23,8 +22,6 @@ import java.util.Map; @Controller("/api/v1/source_oauths") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class SourceOauthApiController implements SourceOauthApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java index 7446ca7f15af..692e35564b9e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java @@ -13,15 +13,12 @@ import io.airbyte.api.model.generated.ConnectionStateCreateOrUpdate; import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.StateHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/state") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class StateApiController implements StateApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java index ca42ae51e2cb..9609ca26ca90 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java @@ -25,15 +25,12 @@ import io.airbyte.commons.server.handlers.WebBackendCheckUpdatesHandler; import io.airbyte.commons.server.handlers.WebBackendConnectionsHandler; import io.airbyte.commons.server.handlers.WebBackendGeographiesHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/web_backend") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) public class WebBackendApiController implements WebBackendApi { @@ -105,6 +102,7 @@ public WebBackendGeographiesListResult webBackendListGeographies() { @Post("/connections/update") @Secured({EDITOR}) + @SecuredWorkspace @Override public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConnectionUpdate webBackendConnectionUpdate) { return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendUpdateConnection(webBackendConnectionUpdate)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java index 1611a7807871..2c628fcd8498 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java @@ -5,7 +5,7 @@ package io.airbyte.server.apis; import static io.airbyte.commons.auth.AuthRoleConstants.AUTHENTICATED_USER; -import static io.airbyte.commons.auth.AuthRoleConstants.OWNER; +import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; import static io.airbyte.commons.auth.AuthRoleConstants.READER; import io.airbyte.api.generated.WorkspaceApi; @@ -20,7 +20,6 @@ import io.airbyte.api.model.generated.WorkspaceUpdateName; import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.WorkspacesHandler; -import io.micronaut.context.annotation.Requires; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; @@ -30,8 +29,6 @@ import io.micronaut.security.rules.SecurityRule; @Controller("/api/v1/workspaces") -@Requires(property = "airbyte.deployment-mode", - value = "OSS") @Secured(SecurityRule.IS_AUTHENTICATED) @SuppressWarnings("PMD.AvoidDuplicateLiterals") public class WorkspaceApiController implements WorkspaceApi { @@ -50,7 +47,7 @@ public WorkspaceRead createWorkspace(@Body final WorkspaceCreate workspaceCreate } @Post("/delete") - @Secured({OWNER}) + @Secured({EDITOR}) @SecuredWorkspace @Override @Status(HttpStatus.NO_CONTENT) @@ -62,7 +59,7 @@ public void deleteWorkspace(@Body final WorkspaceIdRequestBody workspaceIdReques } @Post("/get") - @Secured({OWNER}) + @Secured({READER}) @SecuredWorkspace @Override public WorkspaceRead getWorkspace(@Body final WorkspaceIdRequestBody workspaceIdRequestBody) { @@ -70,7 +67,7 @@ public WorkspaceRead getWorkspace(@Body final WorkspaceIdRequestBody workspaceId } @Post("/get_by_slug") - @Secured({OWNER}) + @Secured({READER}) @SecuredWorkspace @Override public WorkspaceRead getWorkspaceBySlug(@Body final SlugRequestBody slugRequestBody) { @@ -85,7 +82,7 @@ public WorkspaceReadList listWorkspaces() { } @Post("/update") - @Secured({OWNER}) + @Secured({EDITOR}) @SecuredWorkspace @Override public WorkspaceRead updateWorkspace(@Body final WorkspaceUpdate workspaceUpdate) { @@ -93,7 +90,7 @@ public WorkspaceRead updateWorkspace(@Body final WorkspaceUpdate workspaceUpdate } @Post("/tag_feedback_status_as_done") - @Secured({OWNER}) + @Secured({EDITOR}) @SecuredWorkspace @Override public void updateWorkspaceFeedback(@Body final WorkspaceGiveFeedback workspaceGiveFeedback) { @@ -104,7 +101,7 @@ public void updateWorkspaceFeedback(@Body final WorkspaceGiveFeedback workspaceG } @Post("/update_name") - @Secured({OWNER}) + @Secured({EDITOR}) @SecuredWorkspace @Override public WorkspaceRead updateWorkspaceName(@Body final WorkspaceUpdateName workspaceUpdateName) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java index e70d08d72da5..0e75461a1e8a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java @@ -27,6 +27,7 @@ import io.micronaut.core.util.StringUtils; import jakarta.inject.Named; import jakarta.inject.Singleton; +import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Locale; import java.util.UUID; @@ -109,6 +110,11 @@ public AirbyteProtocolVersionRange airbyteProtocolVersionRange( return new AirbyteProtocolVersionRange(new Version(minVersion), new Version(maxVersion)); } + @Singleton + public HttpClient httpClient() { + return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + } + private T convertToEnum(final String value, final Function creatorFunction, final T defaultValue) { return StringUtils.isNotEmpty(value) ? creatorFunction.apply(value.toUpperCase(Locale.ROOT)) : defaultValue; } diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 43dc8c3772e1..b00f5eb07b05 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -7,13 +7,14 @@ import { SynchronousJobRead } from "core/request/AirbyteClient"; import { useAttemptLink } from "./attemptLinkUtils"; import ContentWrapper from "./components/ContentWrapper"; -import ErrorDetails from "./components/ErrorDetails"; -import { JobLogs } from "./components/JobLogs"; import MainInfo from "./components/MainInfo"; import styles from "./JobItem.module.scss"; import { JobsWithJobs } from "./types"; import { didJobSucceed, getJobAttempts, getJobId } from "./utils"; +const ErrorDetails = React.lazy(() => import("./components/ErrorDetails")); +const JobLogs = React.lazy(() => import("./components/JobLogs")); + const Item = styled.div<{ isFailed: boolean }>` border-bottom: 1px solid ${({ theme }) => theme.greyColor20}; font-size: 15px; diff --git a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx index f6043d46b9a8..c01251137781 100644 --- a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx @@ -116,3 +116,5 @@ export const JobLogs: React.FC = ({ jobIsFailed, job }) => { ); }; + +export default JobLogs; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.tsx index 801c954db751..c7e5bb0095ee 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.tsx @@ -3,11 +3,15 @@ import React, { useCallback } from "react"; import { FormattedMessage } from "react-intl"; import { CatalogTree } from "components/connection/CatalogTree"; +import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { SyncSchemaStream } from "core/domain/catalog"; import { DestinationSyncMode } from "core/request/AirbyteClient"; +import { useNewTableDesignExperiment } from "hooks/connection/useNewTableDesignExperiment"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { links } from "utils/links"; import styles from "./SyncCatalogField.module.scss"; @@ -36,13 +40,24 @@ const SyncCatalogFieldComponent: React.FC
- {mode !== "readonly" && additionalControl} + + {newTable && ( + + )} + {mode !== "readonly" && additionalControl} +
diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap index d612bc30ae37..693be2b59629 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap @@ -556,32 +556,36 @@ exports[`CreateConnectionForm should render 1`] = ` > Activate the streams you want to sync - + + Refresh source schema + + +
import("views/Connector/ConnectorForm/components/FrequentlyUsedConnectors") +); interface DestinationFormProps { onSubmit: (values: { diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 5d510e757e6d..9dfd13a2f156 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -55,6 +55,7 @@ "form.cursorField": "Cursor field", "form.dataSync": "Activate the streams you want to sync", "form.dataSync.readonly": "Activated streams", + "form.shareFeedback": "Share Feedback", "form.apply": "Apply", "form.cancel": "Cancel", "form.canceling": "Canceling", diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx index b0200005b586..5e15a1cee4b2 100644 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx @@ -43,3 +43,5 @@ export const InlineEnrollmentCallout: React.FC = () => { ); }; + +export default InlineEnrollmentCallout; diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx index 88cb3a56029f..8992045c73bb 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx @@ -9,6 +9,7 @@ import { ItemTabs, StepsTypes, TableItemTitle } from "components/ConnectorBlocks import LoadingPage from "components/LoadingPage"; import Placeholder, { ResourceTypes } from "components/Placeholder"; import { Breadcrumbs } from "components/ui/Breadcrumbs"; +import type { DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; @@ -18,11 +19,11 @@ import { useGetSource } from "hooks/services/useSourceHook"; import { useSourceDefinition } from "services/connector/SourceDefinitionService"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout"; -import SourceConnectionTable from "./components/SourceConnectionTable"; -import SourceSettings from "./components/SourceSettings"; -import { DropdownMenuOptionType } from "../../../../components/ui/DropdownMenu"; import { RoutePaths } from "../../../routePaths"; +const SourceConnectionTable = React.lazy(() => import("./components/SourceConnectionTable")); +const SourceSettings = React.lazy(() => import("./components/SourceSettings")); + const SourceItemPage: React.FC = () => { useTrackPage(PageTrackingCodes.SOURCE_ITEM); const params = useParams<{ "*": StepsTypes | "" | undefined; id: string }>(); diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx index a58ac3d46ccd..e097e50ed81a 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx @@ -62,9 +62,9 @@ const SourceSettings: React.FC = ({ currentSource, connecti

{connectionsWithSource.map((connection) => ( - <> + - {`${connection.name}\n`} - + ))}

); diff --git a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageTitle.tsx b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageTitle.tsx index 187b2770e0c1..ad278f4e2292 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageTitle.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageTitle.tsx @@ -14,11 +14,14 @@ import { Text } from "components/ui/Text"; import { ConnectionStatus } from "core/request/AirbyteClient"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useFeature, FeatureItem } from "hooks/services/Feature"; -import { InlineEnrollmentCallout } from "packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout"; import styles from "./ConnectionPageTitle.module.scss"; import { ConnectionRoutePaths } from "../types"; +const InlineEnrollmentCallout = React.lazy( + () => import("packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout") +); + export const ConnectionPageTitle: React.FC = () => { const params = useParams<{ id: string; "*": ConnectionRoutePaths }>(); const navigate = useNavigate(); diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap index e5bc00f72d92..8540b10cd748 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap @@ -487,32 +487,36 @@ exports[`ConnectionReplicationPage should render 1`] = ` > Activate the streams you want to sync - + + Refresh source schema +
+ +
{ values={{ count: connectionsWithDestination.length }} /> {connectionsWithDestination.map((connection) => ( - <> + - {`${connection.name}\n`} - + ))}

); diff --git a/airbyte-webapp/src/utils/links.ts b/airbyte-webapp/src/utils/links.ts index ddbdb83ecb62..673f09bcb689 100644 --- a/airbyte-webapp/src/utils/links.ts +++ b/airbyte-webapp/src/utils/links.ts @@ -37,6 +37,7 @@ export const links = { connectionDataResidency: "https://docs.airbyte.com/cloud/managing-airbyte-cloud/#choose-the-data-residency-for-a-connection", lowCodeYamlDescription: `${BASE_DOCS_LINK}/connector-development/config-based/understanding-the-yaml-file/yaml-overview`, + newTableFeedbackUrl: "https://forms.gle/5wDQHf6hL5sCvLgK7", } as const; export type OutboundLinks = typeof links; diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx index caf4bbbde434..592c7fdc8092 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx @@ -21,7 +21,8 @@ import { useAnalyticsTrackFunctions } from "./useAnalyticsTrackFunctions"; import { useTestConnector } from "./useTestConnector"; import { useDocumentationPanelContext } from "../ConnectorDocumentationLayout/DocumentationPanelContext"; import { ConnectorDefinitionTypeControl } from "../ConnectorForm/components/Controls/ConnectorServiceTypeControl"; -import { FetchingConnectorError } from "../ConnectorForm/components/TestingConnectionError"; + +const FetchingConnectorError = React.lazy(() => import("../ConnectorForm/components/TestingConnectionError")); // TODO: need to clean up the ConnectorCard and ConnectorForm props, // since some of props are used in both components, and some of them used just as a prop-drill diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/FrequentlyUsedConnectors/index.ts b/airbyte-webapp/src/views/Connector/ConnectorForm/components/FrequentlyUsedConnectors/index.ts index 64b1515cdfe9..7aa8a596dbf8 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/FrequentlyUsedConnectors/index.ts +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/FrequentlyUsedConnectors/index.ts @@ -1 +1 @@ -export { FrequentlyUsedConnectors } from "./FrequentlyUsedConnectors"; +export { FrequentlyUsedConnectors as default } from "./FrequentlyUsedConnectors"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/TestingConnectionError.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/TestingConnectionError.tsx index e9538929578e..d0a1676c6bc3 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/TestingConnectionError.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/TestingConnectionError.tsx @@ -21,15 +21,15 @@ const ErrorSection: React.FC<{ ); -const TestingConnectionError: React.FC<{ errorMessage: React.ReactNode }> = ({ errorMessage }) => ( +export const TestingConnectionError: React.FC<{ errorMessage: React.ReactNode }> = ({ errorMessage }) => ( } errorMessage={errorMessage} /> ); -const FetchingConnectorError: React.FC = () => ( +export const FetchingConnectorError: React.FC = () => ( } errorMessage={} /> ); -export { TestingConnectionError, FetchingConnectorError }; +export default FetchingConnectorError; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/index.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/index.tsx index 683fe9636173..40bd84fcdf3a 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/index.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/index.tsx @@ -1,4 +1,2 @@ export * from "./types"; export * from "./ConnectorForm"; - -export { FrequentlyUsedConnectors } from "./components/FrequentlyUsedConnectors"; diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 984553e55a34..a7a2805af6a7 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -55,6 +55,7 @@ dependencies { implementation project(':airbyte-analytics') implementation project(':airbyte-api') + implementation project(':airbyte-commons-converters') implementation project(':airbyte-commons-protocol') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-commons-worker') diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java index 8dd971ad8cc5..3f87c5b71d1c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; @@ -30,7 +31,6 @@ import io.airbyte.workers.Worker; import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.general.DefaultCheckConnectionWorker; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.DefaultAirbyteStreamFactory; import io.airbyte.workers.internal.VersionedAirbyteStreamFactory; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java index d34e8f0aa074..2f4fc1da7ae3 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; @@ -30,7 +31,6 @@ import io.airbyte.workers.Worker; import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.general.DefaultDiscoverCatalogWorker; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.VersionedAirbyteStreamFactory; import io.airbyte.workers.process.AirbyteIntegrationLauncher; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index d206ef694f4c..89ef914af7ba 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -17,6 +17,7 @@ import io.airbyte.api.client.model.generated.ConnectionState; import io.airbyte.api.client.model.generated.ConnectionStateType; import io.airbyte.api.client.model.generated.SaveAttemptSyncConfigRequestBody; +import io.airbyte.commons.converters.StateConverter; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.converters.ApiPojoConverters; @@ -45,7 +46,6 @@ import io.airbyte.persistence.job.models.Job; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.WorkerConstants; -import io.airbyte.workers.helper.StateConverter; import io.airbyte.workers.utils.ConfigReplacer; import io.micronaut.context.annotation.Requires; import jakarta.inject.Singleton; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivity.java index e1cc8ccbce75..cdb7d8dc5cf3 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivity.java @@ -4,14 +4,17 @@ package io.airbyte.workers.temporal.sync; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.temporal.activity.ActivityInterface; import io.temporal.activity.ActivityMethod; +import java.util.UUID; @ActivityInterface public interface NormalizationActivity { @@ -24,4 +27,9 @@ NormalizationSummary normalize(JobRunConfig jobRunConfig, @ActivityMethod NormalizationInput generateNormalizationInput(final StandardSyncInput syncInput, final StandardSyncOutput syncOutput); + @ActivityMethod + NormalizationInput generateNormalizationInputWithMinimumPayload(final JsonNode destinationConfiguration, + final ConfiguredAirbyteCatalog airbyteCatalog, + final UUID workspaceId); + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index 8b7ef2f885d0..0f3362edf572 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -9,6 +9,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.DESTINATION_DOCKER_IMAGE_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; @@ -34,6 +35,7 @@ import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.workers.ContainerOrchestratorConfig; import io.airbyte.workers.Worker; import io.airbyte.workers.WorkerConfigs; @@ -176,6 +178,11 @@ public NormalizationSummary normalize(final JobRunConfig jobRunConfig, @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) @Override + @Deprecated(forRemoval = true) + /** + * This activity is deprecated. It is using a big payload which is not needed, it has been replace + * by generateNormalizationInputWithMinimumPayload + */ public NormalizationInput generateNormalizationInput(final StandardSyncInput syncInput, final StandardSyncOutput syncOutput) { return new NormalizationInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) @@ -184,6 +191,18 @@ public NormalizationInput generateNormalizationInput(final StandardSyncInput syn .withWorkspaceId(syncInput.getWorkspaceId()); } + @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) + @Override + public NormalizationInput generateNormalizationInputWithMinimumPayload(final JsonNode destinationConfiguration, + final ConfiguredAirbyteCatalog airbyteCatalog, + final UUID workspaceId) { + return new NormalizationInput() + .withDestinationConfiguration(destinationConfiguration) + .withCatalog(airbyteCatalog) + .withResourceRequirements(normalizationResourceRequirements) + .withWorkspaceId(workspaceId); + } + @VisibleForTesting static boolean normalizationSupportsV1DataTypes(final IntegrationLauncherConfig destinationLauncherConfig) { try { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java index 9d4e50fa9159..4a8adc8fdd15 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java @@ -4,10 +4,10 @@ package io.airbyte.workers.temporal.sync; +import static io.airbyte.commons.converters.StateConverter.convertClientStateTypeToInternal; import static io.airbyte.config.helpers.StateMessageHelper.isMigration; import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; -import static io.airbyte.workers.helper.StateConverter.convertClientStateTypeToInternal; import com.google.common.annotations.VisibleForTesting; import datadog.trace.api.Trace; @@ -15,6 +15,7 @@ import io.airbyte.api.client.model.generated.ConnectionIdRequestBody; import io.airbyte.api.client.model.generated.ConnectionState; import io.airbyte.api.client.model.generated.ConnectionStateCreateOrUpdate; +import io.airbyte.commons.converters.StateConverter; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.State; @@ -25,7 +26,6 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.workers.helper.StateConverter; import jakarta.inject.Singleton; import java.util.List; import java.util.Map; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java index 4a5a1224c8b5..267b43f09182 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java @@ -19,6 +19,7 @@ import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.JobIdRequestBody; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.FeatureFlagHelper; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedSupplier; @@ -57,7 +58,6 @@ import io.airbyte.workers.WorkerMetricReporter; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.general.DefaultReplicationWorker; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteSource; import io.airbyte.workers.internal.DefaultAirbyteDestination; import io.airbyte.workers.internal.DefaultAirbyteSource; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index a77460393f57..32e15285f7cd 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -49,6 +49,8 @@ public class SyncWorkflowImpl implements SyncWorkflow { private static final int NORMALIZATION_SUMMARY_CHECK_CURRENT_VERSION = 1; private static final String AUTO_DETECT_SCHEMA_TAG = "auto_detect_schema"; private static final int AUTO_DETECT_SCHEMA_VERSION = 2; + private static final String USE_MINIMAL_NORM_INPUT = "use_minimal_norm_input"; + private static final int USE_MINIMAL_NORM_INPUT_VERSION = 1; @TemporalActivityStub(activityOptionsBeanName = "longRunActivityOptions") private ReplicationActivity replicationActivity; @TemporalActivityStub(activityOptionsBeanName = "longRunActivityOptions") @@ -186,7 +188,14 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, private NormalizationInput generateNormalizationInput(final StandardSyncInput syncInput, final StandardSyncOutput syncOutput) { - return normalizationActivity.generateNormalizationInput(syncInput, syncOutput); + final int version = Workflow.getVersion(USE_MINIMAL_NORM_INPUT, Workflow.DEFAULT_VERSION, USE_MINIMAL_NORM_INPUT_VERSION); + if (version == Workflow.DEFAULT_VERSION) { + return normalizationActivity.generateNormalizationInput(syncInput, syncOutput); + } else { + return normalizationActivity.generateNormalizationInputWithMinimumPayload(syncInput.getDestinationConfiguration(), + syncOutput.getOutputCatalog(), + syncInput.getWorkspaceId()); + } } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java index bf2a33c78bf8..6a4de155624d 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ActorType; @@ -36,7 +37,6 @@ import io.airbyte.protocol.models.Config; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.test_utils.AirbyteMessageUtils; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java index a0ed43e350ec..0e3d636d00d9 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java @@ -28,6 +28,8 @@ import io.airbyte.api.client.generated.SourceApi; import io.airbyte.api.client.model.generated.DiscoverCatalogResult; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaWriteRequestBody; +import io.airbyte.commons.converters.CatalogClientConverters; +import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; @@ -44,8 +46,6 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.helper.CatalogClientConverters; -import io.airbyte.workers.helper.ConnectorConfigUpdater; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.test_utils.AirbyteMessageUtils; @@ -138,7 +138,7 @@ void testDiscoverSchema() throws Exception { assertNull(output.getFailureReason()); assertEquals(OutputType.DISCOVER_CATALOG_ID, output.getOutputType()); assertEquals(CATALOG_ID, output.getDiscoverCatalogId()); - ArgumentCaptor argument = + final ArgumentCaptor argument = ArgumentCaptor.forClass(SourceDiscoverSchemaWriteRequestBody.class); verify(mSourceApi).writeDiscoverCatalogResult(argument.capture()); assertEquals(CatalogClientConverters.toAirbyteCatalogClientApi(CATALOG), argument.getValue().getCatalog()); @@ -172,7 +172,7 @@ void testDiscoverSchemaWithConfigUpdate() throws Exception { assertNull(output.getFailureReason()); assertEquals(OutputType.DISCOVER_CATALOG_ID, output.getOutputType()); assertEquals(CATALOG_ID, output.getDiscoverCatalogId()); - ArgumentCaptor argument = + final ArgumentCaptor argument = ArgumentCaptor.forClass(SourceDiscoverSchemaWriteRequestBody.class); verify(mSourceApi).writeDiscoverCatalogResult(argument.capture()); assertEquals(CatalogClientConverters.toAirbyteCatalogClientApi(CATALOG), argument.getValue().getCatalog()); @@ -204,7 +204,7 @@ void testDiscoverSchemaWithConfigUpdateNoChange() throws Exception { assertEquals(OutputType.DISCOVER_CATALOG_ID, output.getOutputType()); assertEquals(CATALOG_ID, output.getDiscoverCatalogId()); assertFalse(output.getConnectorConfigurationUpdated()); - ArgumentCaptor argument = + final ArgumentCaptor argument = ArgumentCaptor.forClass(SourceDiscoverSchemaWriteRequestBody.class); verify(mSourceApi).writeDiscoverCatalogResult(argument.capture()); assertEquals(CatalogClientConverters.toAirbyteCatalogClientApi(CATALOG), argument.getValue().getCatalog()); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 73e04f5b9fa8..aebd44e05f8a 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -145,7 +145,7 @@ void setUp() { refreshSchemaActivity = mock(RefreshSchemaActivityImpl.class); configFetchActivity = mock(ConfigFetchActivityImpl.class); - when(normalizationActivity.generateNormalizationInput(any(), any())).thenReturn(normalizationInput); + when(normalizationActivity.generateNormalizationInputWithMinimumPayload(any(), any(), any())).thenReturn(normalizationInput); when(normalizationSummaryCheckActivity.shouldRunNormalization(any(), any(), any())).thenReturn(true); when(configFetchActivity.getSourceId(sync.getConnectionId())).thenReturn(Optional.of(SOURCE_ID)); diff --git a/docs/cli-documentation.md b/docs/cli-documentation.md new file mode 100644 index 000000000000..1df96e2b6f27 --- /dev/null +++ b/docs/cli-documentation.md @@ -0,0 +1,725 @@ +# CLI documentation + +## Disclaimer + +The project is in **alpha** version. +Readers can refer to our [opened GitHub issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Foctavia-cli) to check the ongoing work on this project. + +## What is `octavia` CLI? + +Octavia CLI is a tool to manage Airbyte configurations in YAML. +It has the following features: + +- Scaffolding of a readable directory architecture that will host the YAML configs (`octavia init`). +- Auto-generation of YAML config file that matches the resources' schemas (`octavia generate`). +- Manage Airbyte resources with YAML config files. +- Safe resources update through diff display and validation (`octavia apply`). +- Simple secret management to avoid versioning credentials. + +## Why should I use `octavia` CLI? + +A CLI provides freedom to users to use the tool in whatever context and use case they have. +These are non-exhaustive use cases `octavia` can be convenient for: + +- Managing Airbyte configurations with a CLI instead of a web UI. +- Versioning Airbyte configurations in Git. +- Updating of Airbyte configurations in an automated deployment pipeline. +- Integrating the Airbyte configuration deployment in a dev ops tooling stack: Helm, Ansible etc. +- Streamlining the deployment of Airbyte configurations to multiple Airbyte instance. + +Feel free to share your use cases with the community in [#octavia-cli](https://airbytehq.slack.com/archives/C02RRUG9CP5) or on [Discourse](https://discuss.airbyte.io/). + +## Table of content + +- [Workflow](#workflow) +- [Secret management](#secret-management) +- [Install](#install) +- [Commands reference](#commands-reference) +- [Contributing](#contributing) +- [Telemetry](#telemetry) +- [Changelog](#changelog) + +## Workflow + +### 1. Generate local YAML files for sources or destinations + +1. Retrieve the _definition id_ of the connector you want to use using `octavia list` command. +2. Generate YAML configuration running `octavia generate source ` or `octavia generate destination `. + +### 2. Edit your local YAML configurations + +1. Edit the generated YAML configurations according to your need. +2. Use the [secret management feature](#secret-management) feature to avoid storing credentials in the YAML files. + +### 3. Create the declared sources or destinations on your Airbyte instance + +1. Run `octavia apply` to create the **sources** and **destinations** + +### 4. Generate connections + +1. Run `octavia octavia generate connection --source --destination ` to create a YAML configuration for a new connection. +2. Edit the created configuration file according to your need: change the scheduling or the replicated streams list. + +### 5. Create the declared connections + +1. Run `octavia apply` to create the newly declared connection on your Airbyte instance. + +### 6. Update your configurations + +Changes in your local configurations can be propagated to your Airbyte instance using `octavia apply`. You will be prompted for validation of changes. You can bypass the validation step using the `--force` flag. + +## Secret management + +Sources and destinations configurations have credential fields that you **do not want to store as plain text in your VCS**. +`octavia` offers secret management through environment variables expansion: + +```yaml +configuration: + password: ${MY_PASSWORD} +``` + +If you have set a `MY_PASSWORD` environment variable, `octavia apply` will load its value into the `password` field. + +## Install + +### Requirements + +We decided to package the CLI in a docker image with portability in mind. +**[Please install and run Docker if you are not](https://docs.docker.com/get-docker/)**. + +### As a command available in your bash profile + +```bash +curl -s -o- https://raw.githubusercontent.com/airbytehq/airbyte/master/octavia-cli/install.sh | bash +``` + +This script: + +1. Pulls the [octavia-cli image](https://hub.docker.com/r/airbyte/octavia-cli/tags) from our Docker registry. +2. Creates an `octavia` alias in your profile. +3. Creates a `~/.octavia` file whose values are mapped to the octavia container's environment variables. + +### Using `docker run` + +```bash +touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container +mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.40.32 +``` + +### Using `docker-compose` + +Using octavia in docker-compose could be convenient for automatic `apply` on start-up. + +Add another entry in the services key of your Airbyte `docker-compose.yml` + +```yaml +services: + # . . . + octavia-cli: + image: airbyte/octavia-cli:latest + command: apply --force + env_file: + - ~/.octavia # Use a local env file to store variables that will be mapped the octavia-cli container + volumes: + - :/home/octavia-project + depends_on: + - webapp +``` + +Other commands besides `apply` can be run like so: + +```bash +docker compose run octavia-cli ` +``` + +## Commands reference + +### `octavia` command flags + +| **Flag** | **Description** | **Env Variable** | **Default** | +| ---------------------------------------- | --------------------------------------------------------------------------------- |----------------------------| ------------------------------------------------------ | +| `--airbyte-url` | Airbyte instance URL. | `AIRBYTE_URL` | `http://localhost:8000` | +| `--airbyte-username` | Airbyte instance username (basic auth). | `AIRBYTE_USERNAME` | `airbyte` | +| `--airbyte-password` | Airbyte instance password (basic auth). | `AIRBYTE_PASSWORD` | `password` | +| `--workspace-id` | Airbyte workspace id. | `AIRBYTE_WORKSPACE_ID` | The first workspace id found on your Airbyte instance. | +| `--enable-telemetry/--disable-telemetry` | Enable or disable the sending of telemetry data. | `OCTAVIA_ENABLE_TELEMETRY` | True | +| `--api-http-header` | HTTP Header value pairs passed while calling Airbyte's API | None | None | +| `--api-http-headers-file-path` | Path to the YAML file that contains custom HTTP Headers to send to Airbyte's API. | None | None | + +#### Using custom HTTP headers + +You can set custom HTTP headers to send to Airbyte's API with options: + +```bash +octavia --api-http-header Header-Name Header-Value --api-http-header Header-Name-2 Header-Value-2 list connectors sources +``` + +You can also use a custom YAML file (one is already created on init in `api_http_headers.yaml`) to declare the HTTP headers to send to the API: + +```yaml +headers: + Authorization: Bearer my-secret-token + User-Agent: octavia-cli/0.0.0 +``` + +Environment variable expansion is available in this Yaml file + +```yaml +headers: + Authorization: Bearer ${MY_API_TOKEN} +``` + +**Options based headers are overriding file based headers if an header is declared in both.** + +### `octavia` subcommands + +| **Command** | **Usage** | +| ----------------------------------------- | ------------------------------------------------------------------------------------------ | +| **`octavia init`** | Initialize required directories for the project. | +| **`octavia list connectors sources`** | List all sources connectors available on the remote Airbyte instance. | +| **`octavia list connectors destination`** | List all destinations connectors available on the remote Airbyte instance. | +| **`octavia list workspace sources`** | List existing sources in current the Airbyte workspace. | +| **`octavia list workspace destinations`** | List existing destinations in the current Airbyte workspace. | +| **`octavia list workspace connections`** | List existing connections in the current Airbyte workspace. | +| **`octavia get source`** | Get the JSON representation of an existing source in current the Airbyte workspace. | +| **`octavia get destination`** | Get the JSON representation of an existing destination in the current Airbyte workspace. | +| **`octavia get connection`** | Get the JSON representation of an existing connection in the current Airbyte workspace. | +| **`octavia import all`** | Import all existing sources, destinations and connections to manage them with octavia-cli. | +| **`octavia import source`** | Import an existing source to manage it with octavia-cli. | +| **`octavia import destination`** | Import an existing destination to manage it with octavia-cli. | +| **`octavia import connection`** | Import an existing connection to manage it with octavia-cli. | +| **`octavia generate source`** | Generate a local YAML configuration for a new source. | +| **`octavia generate destination`** | Generate a local YAML configuration for a new destination. | +| **`octavia generate connection`** | Generate a local YAML configuration for a new connection. | +| **`octavia apply`** | Create or update Airbyte remote resources according to local YAML configurations. | + +#### `octavia init` + +The `octavia init` commands scaffolds the required directory architecture for running `octavia generate` and `octavia apply` commands. + +**Example**: + +```bash +$ mkdir my_octavia_project && cd my_octavia_project +$ octavia init +🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace e1f46f7d-5354-4200-aed6-7816015ca54b. +🐙 - Project is not yet initialized. +🔨 - Initializing the project. +✅ - Created the following directories: sources, destinations, connections. +$ ls +connections destinations sources +``` + +#### `octavia list connectors sources` + +List all the source connectors currently available on your Airbyte instance. + +**Example**: + +```bash +$ octavia list connectors sources +NAME DOCKER REPOSITORY DOCKER IMAGE TAG SOURCE DEFINITION ID +Airtable airbyte/source-airtable 0.1.1 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212 +AWS CloudTrail airbyte/source-aws-cloudtrail 0.1.4 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 +Amazon Ads airbyte/source-amazon-ads 0.1.3 c6b0a29e-1da9-4512-9002-7bfd0cba2246 +Amazon Seller Partner airbyte/source-amazon-seller-partner 0.2.16 e55879a8-0ef8-4557-abcf-ab34c53ec460 +``` + +#### `octavia list connectors destinations` + +List all the destinations connectors currently available on your Airbyte instance. + +**Example**: + +```bash +$ octavia list connectors destinations +NAME DOCKER REPOSITORY DOCKER IMAGE TAG DESTINATION DEFINITION ID +Azure Blob Storage airbyte/destination-azure-blob-storage 0.1.3 b4c5d105-31fd-4817-96b6-cb923bfc04cb +Amazon SQS airbyte/destination-amazon-sqs 0.1.0 0eeee7fb-518f-4045-bacc-9619e31c43ea +BigQuery airbyte/destination-bigquery 0.6.11 22f6c74f-5699-40ff-833c-4a879ea40133 +BigQuery (denormalized typed struct) airbyte/destination-bigquery-denormalized 0.2.10 079d5540-f236-4294-ba7c-ade8fd918496 +``` + +#### `octavia list workspace sources` + +List all the sources existing on your targeted Airbyte instance. + +**Example**: + +```bash +$ octavia list workspace sources +NAME SOURCE NAME SOURCE ID +weather OpenWeather c4aa8550-2122-4a33-9a21-adbfaa638544 +``` + +#### `octavia list workspace destinations` + +List all the destinations existing on your targeted Airbyte instance. + +**Example**: + +```bash +$ octavia list workspace destinations +NAME DESTINATION NAME DESTINATION ID +my_db Postgres c0c977c2-48e7-46fe-9f57-576285c26d42 +``` + +#### `octavia list workspace connections` + +List all the connections existing on your targeted Airbyte instance. + +**Example**: + +```bash +$ octavia list workspace connections +NAME CONNECTION ID STATUS SOURCE ID DESTINATION ID +weather_to_pg a4491317-153e-436f-b646-0b39338f9aab active c4aa8550-2122-4a33-9a21-adbfaa638544 c0c977c2-48e7-46fe-9f57-576285c26d42 +``` + +#### `octavia get source or ` + +Get an existing source in current the Airbyte workspace. You can use a source ID or name. + +| **Argument** | **Description** | +| ------------- | ---------------- | +| `SOURCE_ID` | The source id. | +| `SOURCE_NAME` | The source name. | + +**Examples**: + +```bash +$ octavia get source c0c977c2-48e7-46fe-9f57-576285c26d42 +{'connection_configuration': {'key': '**********', + 'start_date': '2010-01-01T00:00:00.000Z', + 'token': '**********'}, + 'name': 'Pokemon', + 'source_definition_id': 'b08e4776-d1de-4e80-ab5c-1e51dad934a2', + 'source_id': 'c0c977c2-48e7-46fe-9f57-576285c26d42', + 'source_name': 'My Poke', + 'workspace_id': 'c4aa8550-2122-4a33-9a21-adbfaa638544'} +``` + +```bash +$ octavia get source "My Poke" +{'connection_configuration': {'key': '**********', + 'start_date': '2010-01-01T00:00:00.000Z', + 'token': '**********'}, + 'name': 'Pokemon', + 'source_definition_id': 'b08e4776-d1de-4e80-ab5c-1e51dad934a2', + 'source_id': 'c0c977c2-48e7-46fe-9f57-576285c26d42', + 'source_name': 'My Poke', + 'workspace_id': 'c4aa8550-2122-4a33-9a21-adbfaa638544'} +``` + +#### `octavia get destination or ` + +Get an existing destination in current the Airbyte workspace. You can use a destination ID or name. + +| **Argument** | **Description** | +| ------------------ | --------------------- | +| `DESTINATION_ID` | The destination id. | +| `DESTINATION_NAME` | The destination name. | + +**Examples**: + +```bash +$ octavia get destination c0c977c2-48e7-46fe-9f57-576285c26d42 +{ + "destinationDefinitionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", + "destinationId": "18102e7c-5160-4000-841b-15e8ec48c301", + "workspaceId": "18102e7c-5160-4000-883a-30bc7cd65601", + "connectionConfiguration": { + "user": "charles" + }, + "name": "pg", + "destinationName": "Postgres" +} +``` + +```bash +$ octavia get destination pg +{ + "destinationDefinitionId": "18102e7c-5160-4000-821f-4d7cfdf87201", + "destinationId": "18102e7c-5160-4000-841b-15e8ec48c301", + "workspaceId": "18102e7c-5160-4000-883a-30bc7cd65601", + "connectionConfiguration": { + "user": "charles" + }, + "name": "string", + "destinationName": "string" +} +``` + +#### `octavia get connection or ` + +Get an existing connection in current the Airbyte workspace. You can use a connection ID or name. + +| **Argument** | **Description** | +| ----------------- | -------------------- | +| `CONNECTION_ID` | The connection id. | +| `CONNECTION_NAME` | The connection name. | + +**Example**: + +```bash +$ octavia get connection c0c977c2-48e7-46fe-9f57-576285c26d42 +{ + "connectionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", + "name": "Poke To PG", + "namespaceDefinition": "source", + "namespaceFormat": "${SOURCE_NAMESPACE}", + "prefix": "string", + "sourceId": "18102e7c-5340-4000-8eaa-4a86f844b101", + "destinationId": "18102e7c-5340-4000-8e58-6bed49c24b01", + "operationIds": [ + "18102e7c-5340-4000-8ef0-f35c05a49a01" + ], + "syncCatalog": { + "streams": [ + { + "stream": { + "name": "string", + "jsonSchema": {}, + "supportedSyncModes": [ + "full_refresh" + ], + "sourceDefinedCursor": false, + "defaultCursorField": [ + "string" + ], + "sourceDefinedPrimaryKey": [ + [ + "string" + ] + ], + "namespace": "string" + }, + "config": { + "syncMode": "full_refresh", + "cursorField": [ + "string" + ], + "destinationSyncMode": "append", + "primaryKey": [ + [ + "string" + ] + ], + "aliasName": "string", + "selected": false + } + } + ] + }, + "schedule": { + "units": 0, + "timeUnit": "minutes" + }, + "status": "active", + "resourceRequirements": { + "cpu_request": "string", + "cpu_limit": "string", + "memory_request": "string", + "memory_limit": "string" + }, + "sourceCatalogId": "18102e7c-5340-4000-85f3-204ab7715801" +} +``` + +```bash +$ octavia get connection "Poke To PG" +{ + "connectionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", + "name": "Poke To PG", + "namespaceDefinition": "source", + "namespaceFormat": "${SOURCE_NAMESPACE}", + "prefix": "string", + "sourceId": "18102e7c-5340-4000-8eaa-4a86f844b101", + "destinationId": "18102e7c-5340-4000-8e58-6bed49c24b01", + "operationIds": [ + "18102e7c-5340-4000-8ef0-f35c05a49a01" + ], + "syncCatalog": { + "streams": [ + { + "stream": { + "name": "string", + "jsonSchema": {}, + "supportedSyncModes": [ + "full_refresh" + ], + "sourceDefinedCursor": false, + "defaultCursorField": [ + "string" + ], + "sourceDefinedPrimaryKey": [ + [ + "string" + ] + ], + "namespace": "string" + }, + "config": { + "syncMode": "full_refresh", + "cursorField": [ + "string" + ], + "destinationSyncMode": "append", + "primaryKey": [ + [ + "string" + ] + ], + "aliasName": "string", + "selected": false + } + } + ] + }, + "schedule": { + "units": 0, + "timeUnit": "minutes" + }, + "status": "active", + "resourceRequirements": { + "cpu_request": "string", + "cpu_limit": "string", + "memory_request": "string", + "memory_limit": "string" + }, + "sourceCatalogId": "18102e7c-5340-4000-85f3-204ab7715801" +} +``` + +#### `octavia import all` + +Import all existing resources (sources, destinations, connections) on your Airbyte instance to manage them with octavia-cli. + +**Examples**: + +```bash +$ octavia import all +🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace b06c6fbb-cadd-4c5c-bdbb-710add7dedb9. +✅ - Imported source poke in sources/poke/configuration.yaml. State stored in sources/poke/state_b06c6fbb-cadd-4c5c-bdbb-710add7dedb9.yaml +⚠️ - Please update any secrets stored in sources/poke/configuration.yaml +✅ - Imported destination Postgres in destinations/postgres/configuration.yaml. State stored in destinations/postgres/state_b06c6fbb-cadd-4c5c-bdbb-710add7dedb9.yaml +⚠️ - Please update any secrets stored in destinations/postgres/configuration.yaml +✅ - Imported connection poke-to-pg in connections/poke_to_pg/configuration.yaml. State stored in connections/poke_to_pg/state_b06c6fbb-cadd-4c5c-bdbb-710add7dedb9.yaml +``` + +You know have local configuration files for all Airbyte resources that were already existing. +You need to edit any secret values that exist in these configuration files as secrets are not imported. +You can edit the configuration files and run `octavia apply` to continue managing them with octavia-cli. + +#### `octavia import destination or ` + +Import an existing destination to manage it with octavia-cli. You can use a destination ID or name. + +| **Argument** | **Description** | +| ------------------ | --------------------- | +| `DESTINATION_ID` | The destination id. | +| `DESTINATION_NAME` | The destination name. | + +#### `octavia import source or ` + +Import an existing source to manage it with octavia-cli. You can use a source ID or name. + +| **Argument** | **Description** | +| ------------- | ---------------- | +| `SOURCE_ID` | The source id. | +| `SOURCE_NAME` | The source name. | + +**Examples**: + +```bash +$ octavia import source poke +🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace 75658e4f-e5f0-4e35-be0c-bdad33226c94. +✅ - Imported source poke in sources/poke/configuration.yaml. State stored in sources/poke/state_75658e4f-e5f0-4e35-be0c-bdad33226c94.yaml +⚠️ - Please update any secrets stored in sources/poke/configuration.yaml +``` + +You know have local configuration file for an Airbyte source that was already existing. +You need to edit any secret value that exist in this configuration as secrets are not imported. +You can edit the configuration and run `octavia apply` to continue managing it with octavia-cli. + +#### `octavia import destination or ` + +Import an existing destination to manage it with octavia-cli. You can use a destination ID or name. + +| **Argument** | **Description** | +| ------------------ | --------------------- | +| `DESTINATION_ID` | The destination id. | +| `DESTINATION_NAME` | The destination name. | + +**Examples**: + +```bash +$ octavia import destination pg +🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace 75658e4f-e5f0-4e35-be0c-bdad33226c94. +✅ - Imported destination pg in destinations/pg/configuration.yaml. State stored in destinations/pg/state_75658e4f-e5f0-4e35-be0c-bdad33226c94.yaml +⚠️ - Please update any secrets stored in destinations/pg/configuration.yaml +``` + +You know have local configuration file for an Airbyte destination that was already existing. +You need to edit any secret value that exist in this configuration as secrets are not imported. +You can edit the configuration and run `octavia apply` to continue managing it with octavia-cli. + +#### `octavia import connection or ` + +Import an existing connection to manage it with octavia-cli. You can use a connection ID or name. + +| **Argument** | **Description** | +| ----------------- | -------------------- | +| `CONNECTION_ID` | The connection id. | +| `CONNECTION_NAME` | The connection name. | + +**Examples**: + +```bash +$ octavia import connection poke-to-pg +🐙 - Octavia is targetting your Airbyte instance running at http://localhost:8000 on workspace 75658e4f-e5f0-4e35-be0c-bdad33226c94. +✅ - Imported connection poke-to-pg in connections/poke-to-pg/configuration.yaml. State stored in connections/poke-to-pg/state_75658e4f-e5f0-4e35-be0c-bdad33226c94.yaml +⚠️ - Please update any secrets stored in connections/poke-to-pg/configuration.yaml +``` + +You know have local configuration file for an Airbyte connection that was already existing. +**N.B.: You first need to import the source and destination used by the connection.** +You can edit the configuration and run `octavia apply` to continue managing it with octavia-cli. + +#### `octavia generate source ` + +Generate a YAML configuration for a source. +The YAML file will be stored at `./sources//configuration.yaml`. + +| **Argument** | **Description** | +| --------------- | --------------------------------------------------------------------------------------------- | +| `DEFINITION_ID` | The source connector definition id. Can be retrieved using `octavia list connectors sources`. | +| `SOURCE_NAME` | The name you want to give to this source in Airbyte. | + +**Example**: + +```bash +$ octavia generate source d8540a80-6120-485d-b7d6-272bca477d9b weather +✅ - Created the source template for weather in ./sources/weather/configuration.yaml. +``` + +#### `octavia generate destination ` + +Generate a YAML configuration for a destination. +The YAML file will be stored at `./destinations//configuration.yaml`. + +| **Argument** | **Description** | +| ------------------ | ------------------------------------------------------------------------------------------------------- | +| `DEFINITION_ID` | The destination connector definition id. Can be retrieved using `octavia list connectors destinations`. | +| `DESTINATION_NAME` | The name you want to give to this destination in Airbyte. | + +**Example**: + +```bash +$ octavia generate destination 25c5221d-dce2-4163-ade9-739ef790f503 my_db +✅ - Created the destination template for my_db in ./destinations/my_db/configuration.yaml. +``` + +#### `octavia generate connection --source --destination ` + +Generate a YAML configuration for a connection. +The YAML file will be stored at `./connections//configuration.yaml`. + +| **Option** | **Required** | **Description** | +| --------------- | ------------ | ------------------------------------------------------------------------------------------ | +| `--source` | Yes | Path to the YAML configuration file of the source you want to create a connection from. | +| `--destination` | Yes | Path to the YAML configuration file of the destination you want to create a connection to. | + +| **Argument** | **Description** | +| ----------------- | -------------------------------------------------------- | +| `CONNECTION_NAME` | The name you want to give to this connection in Airbyte. | + +**Example**: + +```bash +$ octavia generate connection --source sources/weather/configuration.yaml --destination destinations/my_db/configuration.yaml weather_to_pg +✅ - Created the connection template for weather_to_pg in ./connections/weather_to_pg/configuration.yaml. +``` + +#### `octavia apply` + +Create or update the resource on your Airbyte instance according to local configurations found in your octavia project directory. +If the resource was not found on your Airbyte instance, **apply** will **create** the remote resource. +If the resource was found on your Airbyte instance, **apply** will prompt you for validation of the changes and will run an **update** of your resource. +Please note that if a secret field was updated on your configuration, **apply** will run this change without prompt. + +| **Option** | **Required** | **Description** | +| ---------- | ------------ | ------------------------------------------------------------------ | +| `--file` | No | Path to the YAML configuration files you want to create or update. | +| `--force` | No | Run update without prompting for changes validation. | + +**Example**: + +```bash +$ octavia apply +🐙 - weather exists on your Airbyte instance, let's check if we need to update it! +👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed): + E - Value of root['lat'] changed from "46.7603" to "45.7603". +❓ - Do you want to update weather? [y/N]: y +✍️ - Running update because a diff was detected between local and remote resource. +🎉 - Successfully updated weather on your Airbyte instance! +💾 - New state for weather stored at ./sources/weather/state_.yaml. +🐙 - my_db exists on your Airbyte instance, let's check if we need to update it! +😴 - Did not update because no change detected. +🐙 - weather_to_pg exists on your Airbyte instance, let's check if we need to update it! +👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed): + E - Value of root['schedule']['timeUnit'] changed from "days" to "hours". +❓ - Do you want to update weather_to_pg? [y/N]: y +✍️ - Running update because a diff was detected between local and remote resource. +🎉 - Successfully updated weather_to_pg on your Airbyte instance! +💾 - New state for weather_to_pg stored at ./connections/weather_to_pg/state_.yaml. +``` + +## Contributing + +1. Please sign up to [Airbyte's Slack workspace](https://slack.airbyte.io/) and join the `#octavia-cli`. We'll sync up community efforts in this channel. +2. Pick an existing [GitHub issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Foctavia-cli) or **open** a new one to explain what you'd like to implement. +3. Assign the GitHub issue to yourself. +4. Fork Airbyte's repo, code and test thoroughly. +5. Open a PR on our Airbyte repo from your fork. + +### Developing locally + +0. Build the project locally (from the root of Airbyte's repo): `SUB_BUILD=OCTAVIA_CLI ./gradlew build # from the root directory of the repo`. +1. Install Python 3.8.12. We suggest doing it through `pyenv`. +2. Create a virtualenv: `python -m venv .venv`. +3. Activate the virtualenv: `source .venv/bin/activate`. +4. Install dev dependencies: `pip install -e .\[tests\]`. +5. Install `pre-commit` hooks: `pre-commit install`. +6. Run the unittest suite: `pytest --cov=octavia_cli`. Note, a local version of airbyte needs to be running (e.g. `docker compose up` from the root directory of the project) +7. Make sure the build passes (step 0) before opening a PR. + +## Telemetry + +This CLI has some telemetry tooling to send Airbyte some data about the usage of this tool. +We will use this data to improve the CLI and measure its adoption. +The telemetry sends data about: + +- Which command was run (not the arguments or options used). +- Success or failure of the command run and the error type (not the error payload). +- The current Airbyte workspace id if the user has not set the _anonymous data collection_ on their Airbyte instance. + +You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment variable to `False` or using the `--disable-telemetry` flag. + +## Changelog + +| Version | Date | Description | PR | +| ------- | ---------- | ------------------------------------------------------------------------------------- | ----------------------------------------------------------- | +| 0.41.0 | 2022-10-13 | Use Basic Authentication for making API requests | [#17982](https://github.com/airbytehq/airbyte/pull/17982) | +| 0.40.32 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) | +| 0.39.33 | 2022-07-05 | Add `octavia import all` command | [#14374](https://github.com/airbytehq/airbyte/pull/14374) | +| 0.39.32 | 2022-06-30 | Create import command to import and manage existing Airbyte resource from octavia-cli | [#14137](https://github.com/airbytehq/airbyte/pull/14137) | +| 0.39.27 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | +| 0.39.19 | 2022-06-16 | Allow connection management on multiple workspaces | [#13070](https://github.com/airbytehq/airbyte/pull/12727) | +| 0.39.19 | 2022-06-15 | Allow users to set custom HTTP headers | [#12893](https://github.com/airbytehq/airbyte/pull/12893) | +| 0.39.14 | 2022-05-12 | Enable normalization on connection | [#12727](https://github.com/airbytehq/airbyte/pull/12727) | +| 0.37.0 | 2022-05-05 | Use snake case in connection fields | [#12133](https://github.com/airbytehq/airbyte/pull/12133) | +| 0.35.68 | 2022-04-15 | Improve telemetry | [#12072](https://github.com/airbytehq/airbyte/issues/11896) | +| 0.35.68 | 2022-04-12 | Add telemetry | [#11896](https://github.com/airbytehq/airbyte/issues/11896) | +| 0.35.61 | 2022-04-07 | Alpha release | [EPIC](https://github.com/airbytehq/airbyte/issues/10704) | diff --git a/docs/connector-development/config-based/connector-builder-ui.md b/docs/connector-development/config-based/connector-builder-ui.md index 81fa74df17d9..cec9ee2f7202 100644 --- a/docs/connector-development/config-based/connector-builder-ui.md +++ b/docs/connector-development/config-based/connector-builder-ui.md @@ -48,7 +48,7 @@ The output of this UI is a low-code YAML representation of your connector, which Once you're done iterating on your connector in the UI, you'll need to export the low-code YAML representation of the connector to your local filesystem into a connector module. This YAML can be downloaded by clicking the `Download Config` button in the bottom-left. -If you haven't already, create a low-code connector module using the connector generator (see [this YAML tutorial for an example](tutorial/1-create-source.md)) using the name you'd like to use for your connector. For this section, let's assume our connector is called `exchange-rates`. After creating the connector, overwrite the contents of `airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/exchange_rates.yaml` with the YAML you created in the UI. +If you haven't already, create a low-code connector module using the connector generator (see [this YAML tutorial for an example](tutorial/1-create-source.md)) using the name you'd like to use for your connector. For this section, let's assume our connector is called `exchange-rates`. After creating the connector, overwrite the contents of `airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml` with the YAML you created in the UI. ### Building the connector image @@ -73,11 +73,15 @@ The UI contains two main components: the Builder UI where you can fill out input 9. **Page selector** Displays the selected page 10. **Logs view**: Displays the logs emitted by the connector while running + ## Upgrading diff --git a/docs/connector-development/config-based/low-code-cdk-overview.md b/docs/connector-development/config-based/low-code-cdk-overview.md index 9cc57d391a5d..f91d788f67b5 100644 --- a/docs/connector-development/config-based/low-code-cdk-overview.md +++ b/docs/connector-development/config-based/low-code-cdk-overview.md @@ -144,6 +144,6 @@ This section a tutorial that will guide you through the end-to-end process of im For examples of production-ready config-based connectors, refer to: -- [Greenhouse](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-greenhouse) -- [Sendgrid](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml) -- [Sentry](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml) +- [Greenhouse](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/manifest.yaml) +- [Sendgrid](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/manifest.yaml) +- [Sentry](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml) diff --git a/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md b/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md index ad05a6c84d10..466a199e961d 100644 --- a/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md +++ b/docs/connector-development/config-based/tutorial/3-connecting-to-the-API-source.md @@ -4,7 +4,7 @@ We're now ready to start implementing the connector. Over the course of this tutorial, we'll be editing a few files that were generated by the code generator: -- `source-exchange-rates-tutorial/source_exchange_rates_tutorial/exchange_rates_tutorial.yaml`: This is the connector manifest. It describes how the data should be read from the API source, as well as what inputs can be used to configure the connector. +- `source-exchange-rates-tutorial/source_exchange_rates_tutorial/manifest.yaml`: This is the connector manifest. It describes how the data should be read from the API source, as well as what inputs can be used to configure the connector. - `source-exchange_rates-tutorial/integration_tests/configured_catalog.json`: This is the connector's [catalog](../../../understanding-airbyte/beginners-guide-to-catalog.md). It describes what data is available in a source - `source-exchange-rates-tutorial/integration_tests/sample_state.json`: This is a sample state object to be used to test [incremental syncs](../../cdk-python/incremental-stream.md). @@ -18,7 +18,7 @@ We'll also be creating the following files: Let's populate the specification (`spec`) and the configuration (`secrets/config.json`) so the connector can access the access key and base currency. -1. We'll add these properties to the `spec` block in the `source-exchange-rates-tutorial/source_exchange_rates_tutorial/exchange_rates_tutorial.yaml` +1. We'll add these properties to the `spec` block in the `source-exchange-rates-tutorial/source_exchange_rates_tutorial/manifest.yaml` ```yaml spec: @@ -58,7 +58,7 @@ echo '{"access_key": "", "base": "USD"}' > secrets/config.json ## Updating the connector definition -Next, we'll update the connector definition (`source-exchange-rates-tutorial/source_exchange_rates_tutorial/exchange_rates_tutorial.yaml`). It was generated by the code generation script. +Next, we'll update the connector definition (`source-exchange-rates-tutorial/source_exchange_rates_tutorial/manifest.yaml`). It was generated by the code generation script. More details on the connector definition file can be found in the [overview](../low-code-cdk-overview.md) and [connection definition](../understanding-the-yaml-file/yaml-overview.md) sections. Let's fill this out these TODOs with the information found in the [Exchange Rates API docs](https://apilayer.com/marketplace/exchangerates_data-api). diff --git a/docs/connector-development/config-based/tutorial/5-incremental-reads.md b/docs/connector-development/config-based/tutorial/5-incremental-reads.md index 12013844b025..ef0751f9c3f6 100644 --- a/docs/connector-development/config-based/tutorial/5-incremental-reads.md +++ b/docs/connector-development/config-based/tutorial/5-incremental-reads.md @@ -7,7 +7,7 @@ According to the API documentation, we can read the exchange rate for a specific We'll now add a `start_date` property to the connector. -First we'll update the spec block in `source_exchange_rates_tutorial/exchange_rates_tutorial.yaml` +First we'll update the spec block in `source_exchange_rates_tutorial/manifest.yaml` ```yaml spec: @@ -156,7 +156,7 @@ definitions: stream_cursor_field: "date" ``` -The full connector definition should now look like `./source_exchange_rates_tutorial/exchange_rates_tutorial.yaml`: +The full connector definition should now look like `./source_exchange_rates_tutorial/manifest.yaml`: ```yaml version: "0.1.0" diff --git a/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md b/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md index 7df5435e716c..7dd7ed6eb90c 100644 --- a/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md +++ b/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md @@ -114,7 +114,7 @@ requester: <...> error_handler: response_filters: - - error_message_contain: "ignorethisresponse" + - error_message_contains: "ignorethisresponse" action: IGNORE ``` diff --git a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md index dc232ca74485..5c6d42dba9ab 100644 --- a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md @@ -84,6 +84,8 @@ Example of `acceptance-test-config.yml`: ```yaml connector_image: string # Docker image to test, for example 'airbyte/source-pokeapi:0.1.0' base_path: string # Base path for all relative paths, optional, default - ./ +custom_environment_variables: + foo: bar acceptance_tests: # Tests configuration spec: # list of the test inputs bypass_reason: "Explain why you skipped this test" @@ -399,3 +401,13 @@ or prevent network access for this connector entirely allowedHosts: hosts: [] ``` + +## Custom environment variable + +The connector under tests can be run with custom environment variables: +```yaml +connector_image: "airbyte/source-pokeapi" +custom_environment_variables: + my_custom_environment_variable: value +... +``` \ No newline at end of file diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index 9ba8be95a8cb..33892fe5a2c2 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -76,10 +76,11 @@ The [Google API rate limit](https://developers.google.com/sheets/api/limits) is | Version | Date | Pull Request | Subject | | ------- | ---------- | -------------------------------------------------------- | ----------------------------------------------------------------------------- | +| 0.2.32 | 2023-02-13 | [22884](https://github.com/airbytehq/airbyte/pull/22884) | Do not consume http spreadsheets. | | 0.2.31 | 2022-10-09 | [](https://github.com/airbytehq/airbyte/pull/) | Revert 'Add row_id to rows and use as primary key' | | 0.2.30 | 2022-10-09 | [](https://github.com/airbytehq/airbyte/pull/) | Add row_id to rows and use as primary key | | 0.2.21 | 2022-10-04 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Clean instantiation of AirbyteStream | -| 0.2.20 | 2022-10-10 | [17766](https://github.com/airbytehq/airbyte/pull/17766) | Fix null pointer exception when parsing the spreadsheet id. | +| 0.2.20 | 2022-10-10 | [17766](https://github.com/airbytehq/airbyte/pull/17766) | Fix null pointer exception when parsing the spreadsheet id. | | 0.2.19 | 2022-09-29 | [17410](https://github.com/airbytehq/airbyte/pull/17410) | Use latest CDK. | | 0.2.18 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | | 0.2.17 | 2022-08-03 | [15107](https://github.com/airbytehq/airbyte/pull/15107) | Expose Row Batch Size in Connector Specification | diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md index e0486990cf0b..207372bdc1b9 100644 --- a/docs/integrations/sources/pipedrive.md +++ b/docs/integrations/sources/pipedrive.md @@ -1,42 +1,49 @@ # Pipedrive -## Overview +This page contains the setup guide and reference information for the Pipedrive connector. -The Pipedrive connector can be used to sync your Pipedrive data. It supports full refresh sync for Deals, Leads, Activities, ActivityFields, Persons, Pipelines, Stages, Users streams and incremental sync for Activities, Deals, Persons, Pipelines, Stages, Users streams. +## Prerequisites -There was a priority to include at least a single stream of each stream type which is present on Pipedrive, so the list of the supported streams is meant to be easily extendable. By the way, we can only support incremental stream support for the streams listed [there](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents). +* A Pipedrive account; +* An `API token`; +* A `client_id`, `client_secret`, and `refresh_token`. -### Output schema +## Setup guide -Several output streams are available from this source: +The Pipedrive connector accepts two authentication flows: -* [Activities](https://developers.pipedrive.com/docs/api/v1/Activities#getActivities), +### Via API Token Authentication - retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) \(incremental\) +Step 1 - Enable API Token: -* [ActivityFields](https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields) -* [Deals](https://developers.pipedrive.com/docs/api/v1/Deals#getDeals), +If you don't see API next to the `Your companies` section, it's due to the permission sets handled by the company's admin. The company's admin can give you access to your API token by enabling it for you from the Settings in Pipedrive web app. - retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) \(incremental\) +For more information, access [enabling API for company users](https://pipedrive.readme.io/docs/enabling-api-for-company-users). -* [Leads](https://developers.pipedrive.com/docs/api/v1/Leads#getLeads) -* [Persons](https://developers.pipedrive.com/docs/api/v1/Persons#getPersons), - retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) \(incremental\) +Step 2 - Find the API Token: + +You can get the API Token manually from the Pipedrive web app by going to account name (on the top right) > Company settings > Personal preferences > API. + +See [How to find the API Token](https://pipedrive.readme.io/docs/how-to-find-the-api-token) for detailed information. + +### Via OAuth + +Step 1 - Register a Pipedrive app: -* [Pipelines](https://developers.pipedrive.com/docs/api/v1/Pipelines#getPipelines), +Pipedrive allows integrations with its API through **registered apps**. So, to authenticate Airbyte, first you need to create a Pipedrive private app in the marketplace. Follow these [instructions](https://pipedrive.readme.io/docs/marketplace-registering-the-app) to register your integration. - retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) \(incremental\) +Step 2 - Follow the Oauth Authorization flow: -* [Stages](https://developers.pipedrive.com/docs/api/v1/Stages#getStages), +With the registered app, you can follow the authorization flow to obtain the `client_id`, `client_secret`, and `refresh_token` secrets. Pipedrive has documentation about it: https://pipedrive.readme.io/docs/marketplace-oauth-authorization. - retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) \(incremental\) +Step 3 - Configure Airbyte: -* [Users](https://developers.pipedrive.com/docs/api/v1/Users#getUsers), +Now you can fill the fields Client ID, Client Secret, and Refresh Token. Your Pipedrive connector is set up to work with the OAuth authentication. - retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) \(incremental\) +## Supported sync modes -### Features +The Pipedrive connector supports the following sync modes: | Feature | Supported? | | :---------------------------- | :--------- | @@ -46,42 +53,38 @@ Several output streams are available from this source: | SSL connection | Yes | | Namespaces | No | -### Performance considerations -The Pipedrive connector will gracefully handle rate limits. For more information, see [the Pipedrive docs for rate limitations](https://pipedrive.readme.io/docs/core-api-concepts-rate-limiting). +## Supported Streams + +Apart from `Fields` streams, all other streams support incremental. + +* [Activities](https://developers.pipedrive.com/docs/api/v1/Activities#getActivities) -## Getting started +* [ActivityFields](https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields) -### Requirements +* [DealFields](https://developers.pipedrive.com/docs/api/v1/DealFields#getDealFields) -* Pipedrive Account with wright to generate API Token +* [Deals](https://developers.pipedrive.com/docs/api/v1/Deals#getDeals) -### Setup guide +* [Leads](https://developers.pipedrive.com/docs/api/v1/Leads#getLeads) -This connector supports only authentication with API Token. To obtain API Token follow the instructions below: +* [OrganizationFields](https://developers.pipedrive.com/docs/api/v1/OrganizationFields#getOrganizationFields) -#### Enable API: +* [Organizations](https://developers.pipedrive.com/docs/api/v1/Organizations#getOrganizations) -1. Click Manage users from the left-side menu. -2. Click on the Permission sets tab. -3. Choose the set where the user \(who needs the API enabled\) belongs to. -4. Lastly, click on "use API" on the right-hand side section \(you need to scroll down a bit\). +* [PersonFields](https://developers.pipedrive.com/docs/api/v1/PersonFields#getPersonFields) - Now all users who belong in the set that has the API enabled can find their API token under +* [Persons](https://developers.pipedrive.com/docs/api/v1/Persons#getPersons) - Settings > Personal Preferences > API in their Pipedrive web app. +* [Pipelines](https://developers.pipedrive.com/docs/api/v1/Pipelines#getPipelines) -See [Enabling API for company users](https://pipedrive.readme.io/docs/enabling-api-for-company-users) for more info. +* [Stages](https://developers.pipedrive.com/docs/api/v1/Stages#getStages) -#### How to find the API token: +* [Users](https://developers.pipedrive.com/docs/api/v1/Users#getUsers) -1. Account name \(on the top right\) -2. Company settings -3. Personal preferences -4. API -5. Copy API Token +## Performance considerations -See [How to find the API token](https://pipedrive.readme.io/docs/how-to-find-the-api-token) for more info. +The Pipedrive connector will gracefully handle rate limits. For more information, see [the Pipedrive docs for rate limitations](https://pipedrive.readme.io/docs/core-api-concepts-rate-limiting). ## Changelog @@ -99,4 +102,4 @@ See [How to find the API token](https://pipedrive.readme.io/docs/how-to-find-the | 0.1.3 | 2021-08-26 | [5642](https://github.com/airbytehq/airbyte/pull/5642) | Remove date-time from deals stream | | 0.1.2 | 2021-07-23 | [4912](https://github.com/airbytehq/airbyte/pull/4912) | Update money type to support floating point | | 0.1.1 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Update spec.json | -| 0.1.0 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Release Pipedrive connector! | +| 0.1.0 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | 🎉 New source: Pipedrive connector | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index eac290f386c0..dc76776432aa 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -48,7 +48,7 @@ To replicate data from multiple Postgres schemas, re-run the command to grant ac Grant the user read-only access to the relevant tables: ``` -GRANT SELECT ON ALL TABLES IN SCHEMA TO airbyte; +GRANT SELECT ON ALL TABLES IN SCHEMA TO ; ``` Allow user to see tables created in the future: @@ -411,6 +411,8 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.45 | 2022-02-09 | [22221](https://github.com/airbytehq/airbyte/pull/22371) | Ensures that user has required privileges for CDC syncs. | +| | 2022-02-15 | [23028](https://github.com/airbytehq/airbyte/pull/23028) | | | 1.0.44 | 2022-02-06 | [22221](https://github.com/airbytehq/airbyte/pull/22221) | Exclude new set of system tables when using `pg_stat_statements` extension. | | 1.0.43 | 2022-02-06 | [21634](https://github.com/airbytehq/airbyte/pull/21634) | Improve Standard sync performance by caching objects. | | 1.0.42 | 2022-01-23 | [21523](https://github.com/airbytehq/airbyte/pull/21523) | Check for null in cursor values before replacing. | diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 52bdb1838a17..d4f45b6c5161 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -82,6 +82,7 @@ The Stripe connector should not run into Stripe API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.2 | 2023-02-09 | [22659](https://github.com/airbytehq/airbyte/pull/22659) | Set `AvailabilityStrategy` for all streams | | 1.0.1 | 2023-01-27 | [22042](https://github.com/airbytehq/airbyte/pull/22042) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 1.0.0 | 2023-01-25 | [21858](https://github.com/airbytehq/airbyte/pull/21858) | Update the `Subscriptions` and `Invoices` stream schemas | | 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `Payment Intents` stream schema | diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index 6dbfe0440376..9e3f32439f6a 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -151,7 +151,7 @@ A job specific variable overwrites the default sync job variable defined above. Note that Airbyte does not support logging to separate Cloud Storage providers. -Please see [here](https://docs.airbyte.com/deploying-airbyte/on-kubernetes#configure-logs) for more information on configuring Kuberentes logging. +Please see [here](https://docs.airbyte.com/deploying-airbyte/on-kubernetes#configure-logs) for more information on configuring Kubernetes logging. 1. `GCS_LOG_BUCKET` - Define the GCS bucket to store logs. 2. `S3_BUCKET` - Define the S3 bucket to store logs. diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index f4ddd0b6d6fd..60bbe0050643 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -382,9 +382,8 @@ module.exports = { id: "api-documentation", }, { - type: 'link', - label: 'CLI documentation', - href: 'https://github.com/airbytehq/airbyte/blob/master/octavia-cli/README.md', + type: 'doc', + id: "cli-documentation", }, { type: 'category', diff --git a/octavia-cli/octavia_cli/generate/commands.py b/octavia-cli/octavia_cli/generate/commands.py index 19e3a51299c4..c692ece85846 100644 --- a/octavia-cli/octavia_cli/generate/commands.py +++ b/octavia-cli/octavia_cli/generate/commands.py @@ -49,14 +49,14 @@ def destination(ctx: click.Context, definition_id: str, resource_name: str): "source_path", type=click.Path(exists=True, readable=True), required=True, - help="Path to the YAML fine defining your source configuration.", + help="Path to the YAML file defining your source configuration.", ) @click.option( "--destination", "destination_path", type=click.Path(exists=True, readable=True), required=True, - help="Path to the YAML fine defining your destination configuration.", + help="Path to the YAML file defining your destination configuration.", ) @click.pass_context def connection(ctx: click.Context, connection_name: str, source_path: str, destination_path: str): diff --git a/settings.gradle b/settings.gradle index e24aa6c7fd92..2750de1bfff1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -95,6 +95,7 @@ include ':airbyte-test-utils' // airbyte-workers has a lot of dependencies. include ':airbyte-analytics' // transitively used by airbyte-workers. include ':airbyte-commons-temporal' +include ':airbyte-commons-converters' include ':airbyte-commons-worker' include ':airbyte-config:config-persistence' // transitively used by airbyte-workers. include ':airbyte-featureflag' diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_checks.py b/tools/ci_connector_ops/ci_connector_ops/qa_checks.py index f5c76574751f..1fb8619e7562 100644 --- a/tools/ci_connector_ops/ci_connector_ops/qa_checks.py +++ b/tools/ci_connector_ops/ci_connector_ops/qa_checks.py @@ -4,6 +4,8 @@ import sys +from pathlib import Path +from typing import Iterable, Optional, Set, Tuple from ci_connector_ops.utils import Connector @@ -21,15 +23,17 @@ def check_documentation_file_exists(connector: Connector) -> bool: return connector.documentation_file_path.exists() + def check_documentation_follows_guidelines(connector: Connector) -> bool: """Documentation guidelines are defined here https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw""" follows_guidelines = True with open(connector.documentation_file_path) as f: - doc_lines = [l.lower() for l in f.read().splitlines()] + doc_lines = [line.lower() for line in f.read().splitlines()] if not doc_lines[0].startswith("# "): print("The connector name is not used as the main header in the documentation.") follows_guidelines = False - if connector.definition: # We usually don't have a definition if the connector is not published. + # We usually don't have a definition if the connector is not published. + if connector.definition: if doc_lines[0].strip() != f"# {connector.definition['name'].lower()}": print("The connector name is not used as the main header in the documentation.") follows_guidelines = False @@ -37,13 +41,7 @@ def check_documentation_follows_guidelines(connector: Connector) -> bool: print("The connector name is not used as the main header in the documentation.") follows_guidelines = False - expected_sections = [ - "## Prerequisites", - "## Setup guide", - "## Supported sync modes", - "## Supported streams", - "## Changelog" - ] + expected_sections = ["## Prerequisites", "## Setup guide", "## Supported sync modes", "## Supported streams", "## Changelog"] for expected_section in expected_sections: if expected_section.lower() not in doc_lines: @@ -51,6 +49,7 @@ def check_documentation_follows_guidelines(connector: Connector) -> bool: follows_guidelines = False return follows_guidelines + def check_changelog_entry_is_updated(connector: Connector) -> bool: """Check that the changelog entry is updated for the latest connector version in docs/integrations//.md @@ -72,6 +71,7 @@ def check_changelog_entry_is_updated(connector: Connector) -> bool: return True return False + def check_connector_icon_is_available(connector: Connector) -> bool: """Check an SVG icon exists for a connector in in airbyte-config/init/src/main/resources/icons/.svg @@ -84,6 +84,59 @@ def check_connector_icon_is_available(connector: Connector) -> bool: """ return connector.icon_path.exists() + +def read_all_files_in_directory( + directory: Path, ignored_directories: Optional[Set[str]] = None, ignored_filename_patterns: Optional[Set[str]] = None +) -> Iterable[Tuple[str, str]]: + ignored_directories = ignored_directories if ignored_directories is not None else {} + ignored_filename_patterns = ignored_filename_patterns if ignored_filename_patterns is not None else {} + + for path in directory.rglob("*"): + ignore_directory = any([ignored_directory in path.parts for ignored_directory in ignored_directories]) + ignore_filename = any([path.match(ignored_filename_pattern) for ignored_filename_pattern in ignored_filename_patterns]) + ignore = ignore_directory or ignore_filename + if path.is_file() and not ignore: + try: + for line in open(path, "r"): + yield path, line + except UnicodeDecodeError: + print(f"{path} could not be decoded as it is not UTF8.") + continue + + +IGNORED_DIRECTORIES_FOR_HTTPS_CHECKS = { + ".venv", + "tests", + "unit_tests", + "integration_tests", + "build", + "source-file", + ".pytest_cache", + "acceptance_tests_logs", + ".hypothesis", +} + +IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = {"*Test.java", "*.pyc", "*.gz"} +IGNORED_URLS_PREFIX = {"http://json-schema.org", "http://localhost"} + + +def is_comment(line: str, file_path: Path): + language_comments = { + ".py": "#", + ".yml": "#", + ".yaml": "#", + ".java": "//", + ".md": "