Skip to content

Commit

Permalink
Turn on auto-detect schema env variable (#21814)
Browse files Browse the repository at this point in the history 
* Turn on auto-detect schema env variable
Co-authored-by: Edmundo Ruiz Ghanem <[email protected]>
  • Loading branch information
@alovew
alovew authored Jan 25, 2023
1 parent 7a403bb commit e7cc086
Show file tree
Hide file tree
Showing 13 changed files with 264 additions and 18 deletions.
2 changes: 1 addition & 1 deletion .env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,4 +114,4 @@ METRIC_CLIENT=
OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"

USE_STREAM_CAPABLE_STATE=true
AUTO_DETECT_SCHEMA=false
AUTO_DETECT_SCHEMA=true
2 changes: 1 addition & 1 deletion .env.dev
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ SYNC_JOB_MAX_ATTEMPTS=3
SYNC_JOB_MAX_TIMEOUT_DAYS=3
WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
CRON_MICRONAUT_ENVIRONMENTS=control-plane
AUTO_DETECT_SCHEMA=false
AUTO_DETECT_SCHEMA=true

# Sentry
SENTRY_DSN=""
Expand Down
2 changes: 1 addition & 1 deletion airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public boolean useStreamCapableState() {

@Override
public boolean autoDetectSchema() {
return getEnvOrDefault(AUTO_DETECT_SCHEMA, false, Boolean::parseBoolean);
return getEnvOrDefault(AUTO_DETECT_SCHEMA, true, Boolean::parseBoolean);
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1124,7 +1124,7 @@ public int getWorkflowFailureRestartDelaySeconds() {

@Override
public boolean getAutoDetectSchema() {
return getEnvOrDefault(AUTO_DETECT_SCHEMA, false);
return getEnvOrDefault(AUTO_DETECT_SCHEMA, true);
}

@Override
Expand Down
130 changes: 126 additions & 4 deletions ...mponents/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,128 @@ exports[`CreateConnectionForm should render 1`] = `
</div>
</div>
</div>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<label
class="<removed-for-snapshot-test>"
>
Non-breaking schema updates detected
<span
class="<removed-for-snapshot-test>"
>
<span
class="<removed-for-snapshot-test>"
>
<span
class="<removed-for-snapshot-test>"
>
<svg
fill="none"
height="11"
viewBox="0 0 14 14"
width="11"
>
<path
d="M7.00016 13.6663C3.31816 13.6663 0.333496 10.6817 0.333496 6.99967C0.333496 3.31767 3.31816 0.333008 7.00016 0.333008C10.6822 0.333008 13.6668 3.31767 13.6668 6.99967C13.6668 10.6817 10.6822 13.6663 7.00016 13.6663ZM7.00016 12.333C8.41465 12.333 9.77121 11.7711 10.7714 10.7709C11.7716 9.77072 12.3335 8.41416 12.3335 6.99967C12.3335 5.58519 11.7716 4.22863 10.7714 3.22844C9.77121 2.22824 8.41465 1.66634 7.00016 1.66634C5.58567 1.66634 4.22912 2.22824 3.22893 3.22844C2.22873 4.22863 1.66683 5.58519 1.66683 6.99967C1.66683 8.41416 2.22873 9.77072 3.22893 10.7709C4.22912 11.7711 5.58567 12.333 7.00016 12.333ZM6.3335 3.66634H7.66683V4.99967H6.3335V3.66634ZM6.3335 6.33301H7.66683V10.333H6.3335V6.33301Z"
fill="currentColor"
/>
</svg>
</span>
</span>
</span>
</label>
</div>
</div>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
data-testid="nonBreakingChangesPreference"
role="combobox"
>
<span
class="<removed-for-snapshot-test>"
id="react-select-3-live-region"
/>
<span
aria-atomic="false"
aria-live="polite"
aria-relevant="additions text"
class="<removed-for-snapshot-test>"
/>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
Ignore
</div>
</div>
</div>
<input
aria-autocomplete="list"
aria-expanded="false"
aria-haspopup="true"
aria-readonly="true"
class="<removed-for-snapshot-test>"
id="react-select-3-input"
inputmode="none"
role="combobox"
tabindex="0"
value=""
/>
</div>
<div
class="<removed-for-snapshot-test>"
>
<div
aria-hidden="true"
class="<removed-for-snapshot-test>"
>
<svg
aria-hidden="true"
class="<removed-for-snapshot-test>"
data-icon="sort-down"
data-prefix="fas"
focusable="false"
role="img"
viewBox="0 0 320 512"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M311.9 335.1l-132.4 136.8C174.1 477.3 167.1 480 160 480c-7.055 0-14.12-2.702-19.47-8.109l-132.4-136.8C-9.229 317.8 3.055 288 27.66 288h264.7C316.9 288 329.2 317.8 311.9 335.1z"
fill="currentColor"
/>
</svg>
</div>
</div>
</div>
<input
name="nonBreakingChangesPreference"
type="hidden"
value="ignore"
/>
</div>
</div>
</div>
</div>
</div>
<div
Expand Down Expand Up @@ -280,7 +402,7 @@ exports[`CreateConnectionForm should render 1`] = `
>
<span
class="<removed-for-snapshot-test>"
id="react-select-3-live-region"
id="react-select-4-live-region"
/>
<span
aria-atomic="false"
Expand Down Expand Up @@ -313,7 +435,7 @@ exports[`CreateConnectionForm should render 1`] = `
aria-haspopup="true"
aria-readonly="true"
class="<removed-for-snapshot-test>"
id="react-select-3-input"
id="react-select-4-input"
inputmode="none"
role="combobox"
tabindex="0"
Expand Down Expand Up @@ -786,7 +908,7 @@ exports[`CreateConnectionForm should render 1`] = `
>
<span
class="<removed-for-snapshot-test>"
id="react-select-4-live-region"
id="react-select-5-live-region"
/>
<span
aria-atomic="false"
Expand Down Expand Up @@ -825,7 +947,7 @@ exports[`CreateConnectionForm should render 1`] = `
aria-haspopup="true"
aria-readonly="true"
class="<removed-for-snapshot-test>"
id="react-select-4-input"
id="react-select-5-input"
inputmode="none"
role="combobox"
tabindex="0"
Expand Down
2 changes: 2 additions & 0 deletions airbyte-webapp/src/hooks/services/Feature/constants.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { FeatureItem } from "./types";

export const defaultOssFeatures = [
FeatureItem.AllowAutoDetectSchema,
FeatureItem.AllowCustomDBT,
FeatureItem.AllowSync,
FeatureItem.AllowUpdateConnectors,
Expand All @@ -9,6 +10,7 @@ export const defaultOssFeatures = [
];

export const defaultCloudFeatures = [
FeatureItem.AllowAutoDetectSchema,
FeatureItem.AllowOAuthConnector,
FeatureItem.AllowSync,
FeatureItem.AllowChangeDataGeographies,
Expand Down
130 changes: 126 additions & 4 deletions ...nnections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,128 @@ exports[`ConnectionReplicationPage should render 1`] = `
</div>
</div>
</div>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<label
class="<removed-for-snapshot-test>"
>
Non-breaking schema updates detected
<span
class="<removed-for-snapshot-test>"
>
<span
class="<removed-for-snapshot-test>"
>
<span
class="<removed-for-snapshot-test>"
>
<svg
fill="none"
height="11"
viewBox="0 0 14 14"
width="11"
>
<path
d="M7.00016 13.6663C3.31816 13.6663 0.333496 10.6817 0.333496 6.99967C0.333496 3.31767 3.31816 0.333008 7.00016 0.333008C10.6822 0.333008 13.6668 3.31767 13.6668 6.99967C13.6668 10.6817 10.6822 13.6663 7.00016 13.6663ZM7.00016 12.333C8.41465 12.333 9.77121 11.7711 10.7714 10.7709C11.7716 9.77072 12.3335 8.41416 12.3335 6.99967C12.3335 5.58519 11.7716 4.22863 10.7714 3.22844C9.77121 2.22824 8.41465 1.66634 7.00016 1.66634C5.58567 1.66634 4.22912 2.22824 3.22893 3.22844C2.22873 4.22863 1.66683 5.58519 1.66683 6.99967C1.66683 8.41416 2.22873 9.77072 3.22893 10.7709C4.22912 11.7711 5.58567 12.333 7.00016 12.333ZM6.3335 3.66634H7.66683V4.99967H6.3335V3.66634ZM6.3335 6.33301H7.66683V10.333H6.3335V6.33301Z"
fill="currentColor"
/>
</svg>
</span>
</span>
</span>
</label>
</div>
</div>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
data-testid="nonBreakingChangesPreference"
role="combobox"
>
<span
class="<removed-for-snapshot-test>"
id="react-select-3-live-region"
/>
<span
aria-atomic="false"
aria-live="polite"
aria-relevant="additions text"
class="<removed-for-snapshot-test>"
/>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
<div
class="<removed-for-snapshot-test>"
>
Ignore
</div>
</div>
</div>
<input
aria-autocomplete="list"
aria-expanded="false"
aria-haspopup="true"
aria-readonly="true"
class="<removed-for-snapshot-test>"
id="react-select-3-input"
inputmode="none"
role="combobox"
tabindex="0"
value=""
/>
</div>
<div
class="<removed-for-snapshot-test>"
>
<div
aria-hidden="true"
class="<removed-for-snapshot-test>"
>
<svg
aria-hidden="true"
class="<removed-for-snapshot-test>"
data-icon="sort-down"
data-prefix="fas"
focusable="false"
role="img"
viewBox="0 0 320 512"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M311.9 335.1l-132.4 136.8C174.1 477.3 167.1 480 160 480c-7.055 0-14.12-2.702-19.47-8.109l-132.4-136.8C-9.229 317.8 3.055 288 27.66 288h264.7C316.9 288 329.2 317.8 311.9 335.1z"
fill="currentColor"
/>
</svg>
</div>
</div>
</div>
<input
name="nonBreakingChangesPreference"
type="hidden"
value="ignore"
/>
</div>
</div>
</div>
</div>
</div>
<div
Expand Down Expand Up @@ -211,7 +333,7 @@ exports[`ConnectionReplicationPage should render 1`] = `
>
<span
class="<removed-for-snapshot-test>"
id="react-select-3-live-region"
id="react-select-4-live-region"
/>
<span
aria-atomic="false"
Expand Down Expand Up @@ -244,7 +366,7 @@ exports[`ConnectionReplicationPage should render 1`] = `
aria-haspopup="true"
aria-readonly="true"
class="<removed-for-snapshot-test>"
id="react-select-3-input"
id="react-select-4-input"
inputmode="none"
role="combobox"
tabindex="0"
Expand Down Expand Up @@ -717,7 +839,7 @@ exports[`ConnectionReplicationPage should render 1`] = `
>
<span
class="<removed-for-snapshot-test>"
id="react-select-4-live-region"
id="react-select-5-live-region"
/>
<span
aria-atomic="false"
Expand Down Expand Up @@ -756,7 +878,7 @@ exports[`ConnectionReplicationPage should render 1`] = `
aria-haspopup="true"
aria-readonly="true"
class="<removed-for-snapshot-test>"
id="react-select-4-input"
id="react-select-5-input"
inputmode="none"
role="combobox"
tabindex="0"
Expand Down
2 changes: 1 addition & 1 deletion ...tegration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public void testAsyncOrchestratorPodProcess(final String pullPolicy) throws Inte
null,
null,
null,
Map.of(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true", EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, "false"),
Map.of(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true", EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, "true"),
serverPort);

final Map<Integer, Integer> portMap = Map.of(
Expand Down
2 changes: 1 addition & 1 deletion charts/airbyte/templates/env-configmap.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ data:
ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: ""
WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: ""
USE_STREAM_CAPABLE_STATE: "true"
AUTO_DETECT_SCHEMA: "false"
AUTO_DETECT_SCHEMA: "true"
CONTAINER_ORCHESTRATOR_ENABLED: {{ .Values.worker.containerOrchestrator.enabled | quote }}
CONTAINER_ORCHESTRATOR_IMAGE: {{ .Values.worker.containerOrchestrator.image | quote }}
WORKERS_MICRONAUT_ENVIRONMENTS: "control-plane"
Expand Down
Loading

0 comments on commit e7cc086

Please sign in to comment.