diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json index 2856071fa1c8..7f06675ef2c5 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", "name": "Snowflake", "dockerRepository": "airbyte/destination-snowflake", - "dockerImageTag": "0.3.7", + "dockerImageTag": "0.3.8", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json index cdfca896a4a5..5b3b4625c114 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", "name": "Postgres", "dockerRepository": "airbyte/source-postgres", - "dockerImageTag": "0.3.2", + "dockerImageTag": "0.3.3", "documentationUrl": "https://hub.docker.com/r/airbyte/source-postgres", "icon": "postgresql.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 3eebe4eb89b5..a1208e6aa66f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -22,7 +22,7 @@ - destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba name: Snowflake dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.3.7 + dockerImageTag: 0.3.8 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake - destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 name: S3 diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c940199038a7..37476e757799 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -41,7 +41,7 @@ - sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 name: Postgres dockerRepository: airbyte/source-postgres - dockerImageTag: 0.3.2 + dockerImageTag: 0.3.3 documentationUrl: https://hub.docker.com/r/airbyte/source-postgres icon: postgresql.svg - sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01 diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 13d3e4f54b54..70a7a914853a 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.7 +LABEL io.airbyte.version=0.3.8 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index ee7431aa449b..0a39510ba40e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -79,25 +79,38 @@ "title": "Standard Inserts", "additionalProperties": false, "description": "Uses 
INSERT
statements to send batches of records to Snowflake. Easiest (no setup) but not recommended for large production workloads due to slow speed.", - "required": [], - "properties": {} + "required": ["method"], + "properties": { + "method": { + "type": "string", + "enum": ["Standard"], + "default": "Standard" + } + } }, { "title": "AWS S3 Staging", "additionalProperties": false, "description": "Writes large batches of records to a file, uploads the file to S3, then uses 
COPY INTO table
to upload the file. Recommended for large production workloads for better speed and scalability.", "required": [ + "method", "s3_bucket_name", "access_key_id", "secret_access_key" ], "properties": { + "method": { + "type": "string", + "enum": ["S3 Staging"], + "default": "S3 Staging", + "order": 0 + }, "s3_bucket_name": { "title": "S3 Bucket Name", "type": "string", "description": "The name of the staging S3 bucket. Airbyte will write files to this bucket and read them via 
COPY
statements on Snowflake.", "examples": ["airbyte.staging"], - "order": 0 + "order": 1 }, "s3_bucket_region": { "title": "S3 Bucket Region", @@ -121,6 +134,7 @@ "ca-central-1", "cn-north-1", "cn-northwest-1", + "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", @@ -129,21 +143,21 @@ "sa-east-1", "me-south-1" ], - "order": 1 + "order": 2 }, "access_key_id": { "type": "string", "description": "The Access Key Id granting allow one to access the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket.", "title": "S3 Key Id", "airbyte_secret": true, - "order": 2 + "order": 3 }, "secret_access_key": { "type": "string", "description": "The corresponding secret to the above access key id.", "title": "S3 Access Key", "airbyte_secret": true, - "order": 3 + "order": 4 } } }, @@ -151,21 +165,32 @@ "title": "GCS Staging", "additionalProperties": false, "description": "Writes large batches of records to a file, uploads the file to GCS, then uses 
COPY INTO table
to upload the file. Recommended for large production workloads for better speed and scalability.", - "required": ["project_id", "bucket_name", "credentials_json"], + "required": [ + "method", + "project_id", + "bucket_name", + "credentials_json" + ], "properties": { + "method": { + "type": "string", + "enum": ["GCS Staging"], + "default": "GCS Staging", + "order": 0 + }, "project_id": { "title": "GCP Project ID", "type": "string", "description": "The name of the GCP project ID for your credentials.", "examples": ["my-project"], - "order": 0 + "order": 1 }, "bucket_name": { "title": "GCS Bucket Name", "type": "string", "description": "The name of the staging GCS bucket. Airbyte will write files to this bucket and read them via 
COPY
statements on Snowflake.", "examples": ["airbyte-staging"], - "order": 1 + "order": 2 }, "credentials_json": { "title": "Google Application Credentials", diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 56637fd8ebb3..1b5e86167606 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.2 +LABEL io.airbyte.version=0.3.3 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index 2ef20318d543..878c4bb3db48 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -59,22 +59,37 @@ "title": "Standard", "additionalProperties": false, "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": [], - "properties": {} + "required": ["method"], + "properties": { + "method": { + "type": "string", + "enum": ["Standard"], + "default": "Standard", + "order": 0 + } + } }, { "title": "Logical Replication (CDC)", "additionalProperties": false, "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the Postgres Source docs for more information.", - "required": ["replication_slot", "publication"], + "required": ["method", "replication_slot", "publication"], "properties": { + "method": { + "type": "string", + "enum": ["CDC"], + "default": "CDC", + "order": 0 + }, "replication_slot": { "type": "string", - "description": "A pgoutput logical replication slot." + "description": "A pgoutput logical replication slot.", + "order": 1 }, "publication": { "type": "string", - "description": "A Postgres publication used for consuming changes." + "description": "A Postgres publication used for consuming changes.", + "order": 2 } } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java index e1cc86718976..29df0de4a960 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java @@ -53,7 +53,7 @@ public class PostgresSpecTest { + "\"port\" : 5432, " + "\"host\" : \"localhost\", " + "\"ssl\" : true, " - + "\"replication_method\" : { \"replication_slot\" : \"ab_slot\", \"publication\" : \"ab_publication\" }" + + "\"replication_method\" : { \"method\" : \"CDC\", \"replication_slot\" : \"ab_slot\", \"publication\" : \"ab_publication\" }" + "}"; private static JsonNode schema; private static JsonSchemaValidator validator; @@ -100,6 +100,7 @@ void testWithReplicationMethodStandard() { final JsonNode config = Jsons.deserialize(CONFIGURATION); ((ObjectNode) config.get("replication_method")).remove("replication_slot"); ((ObjectNode) config.get("replication_method")).remove("publication"); + ((ObjectNode) config.get("replication_method")).put("method", "Standard"); assertTrue(validator.test(schema, config)); final JsonNode configReplicationMethodNotSet = Jsons.deserialize(CONFIGURATION);