From cc3990f99f55c7f7ef0f54da1a9ee80f92aa73e8 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 12 Jan 2023 16:36:39 -0800 Subject: [PATCH 1/3] Add AllowedHosts to actor_definitions and database --- .../io/airbyte/bootloader/BootloaderTest.java | 8 ++-- .../main/resources/types/AllowedHosts.yaml | 8 ++++ .../types/StandardDestinationDefinition.yaml | 2 + .../types/StandardSourceDefinition.yaml | 2 + .../config/persistence/ConfigWriter.java | 14 +++++-- .../resources/seed/source_definitions.yaml | 1 + .../V0_40_27_001__AddAllowedHosts.java | 39 +++++++++++++++++++ .../configs_database/schema_dump.txt | 1 + 8 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_27_001__AddAllowedHosts.java diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 691bec931b83..4939a3e2ea6b 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -80,6 +80,10 @@ class BootloaderTest { private static final String VERSION_0321_ALPHA = "0.32.1-alpha"; private static final String VERSION_0170_ALPHA = "0.17.0-alpha"; + // ⚠️ This line should change with every new migration to show that you meant to make a new + // migration to the prod database + private static final String CURRENT_MIGRATION_VERSION = "0.40.27.001"; + @BeforeEach void setup() { container = new PostgreSQLContainer<>("postgres:13-alpine") @@ -146,9 +150,7 @@ void testBootloaderAppBlankDb() throws Exception { assertEquals("0.40.26.001", jobsMigrator.getLatestMigration().getVersion().getVersion()); val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway); - // this line should change with every new migration - // to show that you meant to make a new migration to the prod database - assertEquals("0.40.23.002", configsMigrator.getLatestMigration().getVersion().getVersion()); + assertEquals(CURRENT_MIGRATION_VERSION, configsMigrator.getLatestMigration().getVersion().getVersion()); assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get()); assertEquals(new Version(PROTOCOL_VERSION_123), jobsPersistence.getAirbyteProtocolVersionMin().get()); diff --git a/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml b/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml new file mode 100644 index 000000000000..0524745ebf9c --- /dev/null +++ b/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml @@ -0,0 +1,8 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/AllowedHosts.yaml +title: AllowedHosts +type: array +description: An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted. +items: + type: string diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml index 103530d33b50..2b3fd7e09c4c 100644 --- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml @@ -63,3 +63,5 @@ properties: supportsDbt: type: boolean description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used. + allowedHosts: + "$ref": AllowedHosts.yaml diff --git a/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml index c945477ddbea..905670ac4402 100644 --- a/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml @@ -65,3 +65,5 @@ properties: protocolVersion: type: string description: the Airbyte Protocol version supported by the connector + allowedHosts: + "$ref": AllowedHosts.yaml diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java index 8d8ea240a2f1..17965117af4e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java @@ -105,6 +105,8 @@ static void writeStandardSourceDefinition(final List c .set(Tables.ACTOR_DEFINITION.RESOURCE_REQUIREMENTS, standardSourceDefinition.getResourceRequirements() == null ? null : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getResourceRequirements()))) + .set(ACTOR_DEFINITION.ALLOWED_HOSTS, standardSourceDefinition.getAllowedHosts() == null ? null + : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getAllowedHosts()))) .set(Tables.ACTOR_DEFINITION.UPDATED_AT, timestamp) .where(Tables.ACTOR_DEFINITION.ID.eq(standardSourceDefinition.getSourceDefinitionId())) .execute(); @@ -136,6 +138,8 @@ static void writeStandardSourceDefinition(final List c .set(Tables.ACTOR_DEFINITION.RESOURCE_REQUIREMENTS, standardSourceDefinition.getResourceRequirements() == null ? null : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getResourceRequirements()))) + .set(ACTOR_DEFINITION.ALLOWED_HOSTS, standardSourceDefinition.getAllowedHosts() == null ? null + : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getAllowedHosts()))) .set(Tables.ACTOR_DEFINITION.CREATED_AT, timestamp) .set(Tables.ACTOR_DEFINITION.UPDATED_AT, timestamp) .execute(); @@ -172,7 +176,6 @@ static void writeStandardDestinationDefinition(final List Date: Thu, 12 Jan 2023 16:52:37 -0800 Subject: [PATCH 2/3] use objects for better null-ness handling --- .../src/main/resources/types/AllowedHosts.yaml | 14 ++++++++++---- .../main/resources/seed/source_definitions.yaml | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml b/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml index 0524745ebf9c..3079bca0bd3e 100644 --- a/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml +++ b/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml @@ -2,7 +2,13 @@ "$schema": http://json-schema.org/draft-07/schema# "$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/AllowedHosts.yaml title: AllowedHosts -type: array -description: An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted. -items: - type: string +description: A connector's allowed hosts. If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`. +type: object +required: +additionalProperties: true +properties: + hosts: + type: array + description: An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted. + items: + type: string diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2b66b25ec90f..b279bfb390db 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -501,7 +501,8 @@ icon: faker.svg sourceType: api releaseStage: alpha - allowedHosts: [] + allowedHosts: + hosts: [] - name: Fastbill sourceDefinitionId: eb3e9c1c-0467-4eb7-a172-5265e04ccd0a dockerRepository: airbyte/source-fastbill From cac0426da7b9187caecab9a1552f92b13ab4db9f Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 13 Jan 2023 12:00:09 -0800 Subject: [PATCH 3/3] Tables.ACTOR_DEFINITIO --- .../main/java/io/airbyte/config/persistence/ConfigWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java index 17965117af4e..662ad2562bcc 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java @@ -105,7 +105,7 @@ static void writeStandardSourceDefinition(final List c .set(Tables.ACTOR_DEFINITION.RESOURCE_REQUIREMENTS, standardSourceDefinition.getResourceRequirements() == null ? null : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getResourceRequirements()))) - .set(ACTOR_DEFINITION.ALLOWED_HOSTS, standardSourceDefinition.getAllowedHosts() == null ? null + .set(Tables.ACTOR_DEFINITION.ALLOWED_HOSTS, standardSourceDefinition.getAllowedHosts() == null ? null : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getAllowedHosts()))) .set(Tables.ACTOR_DEFINITION.UPDATED_AT, timestamp) .where(Tables.ACTOR_DEFINITION.ID.eq(standardSourceDefinition.getSourceDefinitionId()))