From f9bef162891af6c964e282bd679ac40eb1f42e2c Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Fri, 13 Jan 2023 13:30:52 -0800 Subject: [PATCH] Add AllowedHosts to Actor Definitions and Config Database (#21363) * Add AllowedHosts to actor_definitions and database * use objects for better null-ness handling * Tables.ACTOR_DEFINITIO --- .../io/airbyte/bootloader/BootloaderTest.java | 8 ++-- .../main/resources/types/AllowedHosts.yaml | 14 +++++++ .../types/StandardDestinationDefinition.yaml | 2 + .../types/StandardSourceDefinition.yaml | 2 + .../config/persistence/ConfigWriter.java | 14 +++++-- .../resources/seed/source_definitions.yaml | 2 + .../V0_40_27_001__AddAllowedHosts.java | 39 +++++++++++++++++++ .../configs_database/schema_dump.txt | 1 + 8 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_27_001__AddAllowedHosts.java diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 691bec931b83..4939a3e2ea6b 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -80,6 +80,10 @@ class BootloaderTest { private static final String VERSION_0321_ALPHA = "0.32.1-alpha"; private static final String VERSION_0170_ALPHA = "0.17.0-alpha"; + // ⚠️ This line should change with every new migration to show that you meant to make a new + // migration to the prod database + private static final String CURRENT_MIGRATION_VERSION = "0.40.27.001"; + @BeforeEach void setup() { container = new PostgreSQLContainer<>("postgres:13-alpine") @@ -146,9 +150,7 @@ void testBootloaderAppBlankDb() throws Exception { assertEquals("0.40.26.001", jobsMigrator.getLatestMigration().getVersion().getVersion()); val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway); - // this line should change with every new migration - // to show that you meant to make a new migration to the prod database - assertEquals("0.40.23.002", configsMigrator.getLatestMigration().getVersion().getVersion()); + assertEquals(CURRENT_MIGRATION_VERSION, configsMigrator.getLatestMigration().getVersion().getVersion()); assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get()); assertEquals(new Version(PROTOCOL_VERSION_123), jobsPersistence.getAirbyteProtocolVersionMin().get()); diff --git a/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml b/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml new file mode 100644 index 000000000000..3079bca0bd3e --- /dev/null +++ b/airbyte-config/config-models/src/main/resources/types/AllowedHosts.yaml @@ -0,0 +1,14 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/AllowedHosts.yaml +title: AllowedHosts +description: A connector's allowed hosts. If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`. +type: object +required: +additionalProperties: true +properties: + hosts: + type: array + description: An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted. + items: + type: string diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml index 103530d33b50..2b3fd7e09c4c 100644 --- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml @@ -63,3 +63,5 @@ properties: supportsDbt: type: boolean description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used. + allowedHosts: + "$ref": AllowedHosts.yaml diff --git a/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml index c945477ddbea..905670ac4402 100644 --- a/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml @@ -65,3 +65,5 @@ properties: protocolVersion: type: string description: the Airbyte Protocol version supported by the connector + allowedHosts: + "$ref": AllowedHosts.yaml diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java index 8d8ea240a2f1..662ad2562bcc 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java @@ -105,6 +105,8 @@ static void writeStandardSourceDefinition(final List c .set(Tables.ACTOR_DEFINITION.RESOURCE_REQUIREMENTS, standardSourceDefinition.getResourceRequirements() == null ? null : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getResourceRequirements()))) + .set(Tables.ACTOR_DEFINITION.ALLOWED_HOSTS, standardSourceDefinition.getAllowedHosts() == null ? null + : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getAllowedHosts()))) .set(Tables.ACTOR_DEFINITION.UPDATED_AT, timestamp) .where(Tables.ACTOR_DEFINITION.ID.eq(standardSourceDefinition.getSourceDefinitionId())) .execute(); @@ -136,6 +138,8 @@ static void writeStandardSourceDefinition(final List c .set(Tables.ACTOR_DEFINITION.RESOURCE_REQUIREMENTS, standardSourceDefinition.getResourceRequirements() == null ? null : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getResourceRequirements()))) + .set(ACTOR_DEFINITION.ALLOWED_HOSTS, standardSourceDefinition.getAllowedHosts() == null ? null + : JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getAllowedHosts()))) .set(Tables.ACTOR_DEFINITION.CREATED_AT, timestamp) .set(Tables.ACTOR_DEFINITION.UPDATED_AT, timestamp) .execute(); @@ -172,7 +176,6 @@ static void writeStandardDestinationDefinition(final List