From e7cc08639480a2a1922ceec50b002edb37578b82 Mon Sep 17 00:00:00 2001 From: Anne <102554163+alovew@users.noreply.github.com> Date: Wed, 25 Jan 2023 13:39:15 -0800 Subject: [PATCH] Turn on auto-detect schema env variable (#21814) * Turn on auto-detect schema env variable Co-authored-by: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> --- .env | 2 +- .env.dev | 2 +- .../features/EnvVariableFeatureFlags.java | 2 +- .../java/io/airbyte/config/EnvConfigs.java | 2 +- .../CreateConnectionForm.test.tsx.snap | 130 +++++++++++++++++- .../src/hooks/services/Feature/constants.ts | 2 + .../ConnectionReplicationPage.test.tsx.snap | 130 +++++++++++++++++- ...OrchestratorPodProcessIntegrationTest.java | 2 +- charts/airbyte/templates/env-configmap.yaml | 2 +- kube/overlays/dev-integration-test/.env | 2 +- kube/overlays/dev/.env | 2 +- .../overlays/stable-with-resource-limits/.env | 2 +- kube/overlays/stable/.env | 2 +- 13 files changed, 264 insertions(+), 18 deletions(-) diff --git a/.env b/.env index d9bf9f7d2048..f15310773da3 100644 --- a/.env +++ b/.env @@ -114,4 +114,4 @@ METRIC_CLIENT= OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317" USE_STREAM_CAPABLE_STATE=true -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/.env.dev b/.env.dev index 4f25503451fc..19480e7072f9 100644 --- a/.env.dev +++ b/.env.dev @@ -28,7 +28,7 @@ SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 WORKERS_MICRONAUT_ENVIRONMENTS=control-plane CRON_MICRONAUT_ENVIRONMENTS=control-plane -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true # Sentry SENTRY_DSN="" diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java index 248c5524ed56..b574d0a40cee 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java @@ -41,7 +41,7 @@ public boolean useStreamCapableState() { @Override public boolean autoDetectSchema() { - return getEnvOrDefault(AUTO_DETECT_SCHEMA, false, Boolean::parseBoolean); + return getEnvOrDefault(AUTO_DETECT_SCHEMA, true, Boolean::parseBoolean); } @Override diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index b68be54b055d..c939764eab65 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -1124,7 +1124,7 @@ public int getWorkflowFailureRestartDelaySeconds() { @Override public boolean getAutoDetectSchema() { - return getEnvOrDefault(AUTO_DETECT_SCHEMA, false); + return getEnvOrDefault(AUTO_DETECT_SCHEMA, true); } @Override diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap index a59078873892..5a7124860480 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap @@ -215,6 +215,128 @@ exports[`CreateConnectionForm should render 1`] = ` +
+
+
+ +
+
+
+
+ + +
+
+
+
+
+ Ignore +
+
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+ +
+
+
+
+ + +
+
+
+
+
+ Ignore +
+
+
+ +
+
+ +
+
+ +
+
+
portMap = Map.of( diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index a219395275e4..f55fd2e88e8b 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -63,7 +63,7 @@ data: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: "" USE_STREAM_CAPABLE_STATE: "true" - AUTO_DETECT_SCHEMA: "false" + AUTO_DETECT_SCHEMA: "true" CONTAINER_ORCHESTRATOR_ENABLED: {{ .Values.worker.containerOrchestrator.enabled | quote }} CONTAINER_ORCHESTRATOR_IMAGE: {{ .Values.worker.containerOrchestrator.image | quote }} WORKERS_MICRONAUT_ENVIRONMENTS: "control-plane" diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index c0d8b1754bde..aad4911e7fda 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -95,4 +95,4 @@ USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index 8762e2ea5914..1bc8b49dfe33 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -96,4 +96,4 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 63df173366ca..22203d6db585 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -98,4 +98,4 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 46400b8ceeda..91ab84ac2d9b 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -96,4 +96,4 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true