From 6a48da9226132107fc702cb74a9a421d5bd62949 Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Fri, 14 Oct 2022 02:59:02 +0200 Subject: [PATCH] Introduce webhook configs into workspace api and persistence (#17950) * wip * handle webhook configs in workspaces endpoint and split/hydrate secrets * style improvements to documentation around webhook configs * Clarify documentation around webhook auth tokens * More documentation clarification around webhook configs * Format. * unit test coverage for webhook config handling * use common json parsing libraries around webhook configs * clean up around testing webhook operation configs Co-authored-by: Davin Chia --- airbyte-api/src/main/openapi/config.yaml | 33 +++++++++ .../java/io/airbyte/commons/json/Jsons.java | 8 +++ .../java/io/airbyte/config/ConfigSchema.java | 3 + .../resources/types/StandardWorkspace.yaml | 6 ++ .../types/WebhookOperationConfigs.yaml | 29 ++++++++ .../DatabaseConfigPersistence.java | 4 ++ .../persistence/SecretsRepositoryReader.java | 3 +- .../persistence/SecretsRepositoryWriter.java | 43 +++++++++--- .../SecretsRepositoryWriterTest.java | 46 ++++++++++++ .../WebhookOperationConfigsConverter.java | 52 ++++++++++++++ .../server/handlers/WorkspacesHandler.java | 18 ++++- .../handlers/WorkspacesHandlerTest.java | 57 +++++++++++++-- .../api/generated-api-html/index.html | 70 +++++++++++++++++++ 13 files changed, 354 insertions(+), 18 deletions(-) create mode 100644 airbyte-config/config-models/src/main/resources/types/WebhookOperationConfigs.yaml create mode 100644 airbyte-server/src/main/java/io/airbyte/server/converters/WebhookOperationConfigsConverter.java diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index a5ef224bc982..e5adea49f375 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2280,6 +2280,22 @@ components: type: boolean defaultGeography: $ref: "#/components/schemas/Geography" + webhookConfigs: + type: array + items: + $ref: "#/components/schemas/WebhookConfigWrite" + WebhookConfigWrite: + type: object + properties: + name: + type: string + description: human readable name for this webhook e.g. for UI display. + authToken: + type: string + description: an auth token, to be passed as the value for an HTTP Authorization header. + validationUrl: + type: string + description: if supplied, the webhook config will be validated by checking that this URL returns a 2xx response. Notification: type: object required: @@ -2384,6 +2400,23 @@ components: type: boolean defaultGeography: $ref: "#/components/schemas/Geography" + webhookConfigs: + type: array + items: + # Note: this omits any sensitive info e.g. auth token + $ref: "#/components/schemas/WebhookConfigRead" + WebhookConfigRead: + type: object + description: the readable info for a webhook config; omits sensitive info e.g. auth token + required: + - id + properties: + id: + type: string + format: uuid + name: + type: string + description: human-readable name e.g. for display in UI WorkspaceUpdateName: type: object required: diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 9ff9158135d4..a92e2c49985c 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -16,10 +16,12 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.stream.MoreStreams; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -39,6 +41,8 @@ public class Jsons { // Object Mapper is thread-safe private static final ObjectMapper OBJECT_MAPPER = MoreMappers.initMapper(); + + private static final ObjectMapper YAML_OBJECT_MAPPER = MoreMappers.initYamlMapper(new YAMLFactory()); private static final ObjectWriter OBJECT_WRITER = OBJECT_MAPPER.writer(new JsonPrettyPrinter()); public static String serialize(final T object) { @@ -89,6 +93,10 @@ public static JsonNode jsonNode(final T object) { return OBJECT_MAPPER.valueToTree(object); } + public static JsonNode jsonNodeFromFile(final File file) throws IOException { + return YAML_OBJECT_MAPPER.readTree(file); + } + public static JsonNode emptyObject() { return jsonNode(Collections.emptyMap()); } diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/ConfigSchema.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/ConfigSchema.java index 84fa66740261..b2f05d26a4e5 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/ConfigSchema.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/ConfigSchema.java @@ -23,6 +23,9 @@ public enum ConfigSchema implements AirbyteConfig { workspaceServiceAccount -> workspaceServiceAccount.getWorkspaceId().toString(), "workspaceId"), + WORKSPACE_WEBHOOK_OPERATION_CONFIGS("WebhookOperationConfigs.yaml", + WebhookOperationConfigs.class), + // source STANDARD_SOURCE_DEFINITION("StandardSourceDefinition.yaml", StandardSourceDefinition.class, diff --git a/airbyte-config/config-models/src/main/resources/types/StandardWorkspace.yaml b/airbyte-config/config-models/src/main/resources/types/StandardWorkspace.yaml index dd65857f7dbd..03c786a207a0 100644 --- a/airbyte-config/config-models/src/main/resources/types/StandardWorkspace.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StandardWorkspace.yaml @@ -50,3 +50,9 @@ properties: type: boolean defaultGeography: "$ref": Geography.yaml + webhookOperationConfigs: + description: + Configurations for webhooks operations, stored as a JSON object so we can replace sensitive info with + coordinates in the secrets manager. Must conform to WebhookOperationConfigs.yaml. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-config/config-models/src/main/resources/types/WebhookOperationConfigs.yaml b/airbyte-config/config-models/src/main/resources/types/WebhookOperationConfigs.yaml new file mode 100644 index 000000000000..1aca08ee2e7c --- /dev/null +++ b/airbyte-config/config-models/src/main/resources/types/WebhookOperationConfigs.yaml @@ -0,0 +1,29 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/WebhookOperationConfigs.yaml +title: WebhookOperationConfigs +description: List of configurations for webhook operations +additionalProperties: false +# NOTE: we have an extra layer of object nesting because the generator has some weird behavior with arrays. +# See https://github.com/OpenAPITools/openapi-generator/issues/7802. +type: object +properties: + webhookConfigs: + type: array + items: + type: object + required: + - id + - name + - authToken + properties: + id: + type: string + format: uuid + name: + type: string + description: human readable name for this webhook e.g., for UI display + authToken: + type: string + airbyte_secret: true + description: An auth token, to be passed as the value for an HTTP Authorization header. Note - must include prefix such as "Bearer ". diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java index e512ae14f72d..725815ea62e8 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java @@ -766,6 +766,8 @@ private void writeStandardWorkspace(final List configs, final .set(WORKSPACE.FIRST_SYNC_COMPLETE, standardWorkspace.getFirstCompletedSync()) .set(WORKSPACE.FEEDBACK_COMPLETE, standardWorkspace.getFeedbackDone()) .set(WORKSPACE.UPDATED_AT, timestamp) + .set(WORKSPACE.WEBHOOK_OPERATION_CONFIGS, standardWorkspace.getWebhookOperationConfigs() == null ? null + : JSONB.valueOf(Jsons.serialize(standardWorkspace.getWebhookOperationConfigs()))) .where(WORKSPACE.ID.eq(standardWorkspace.getWorkspaceId())) .execute(); } else { @@ -786,6 +788,8 @@ private void writeStandardWorkspace(final List configs, final .set(WORKSPACE.FEEDBACK_COMPLETE, standardWorkspace.getFeedbackDone()) .set(WORKSPACE.CREATED_AT, timestamp) .set(WORKSPACE.UPDATED_AT, timestamp) + .set(WORKSPACE.WEBHOOK_OPERATION_CONFIGS, standardWorkspace.getWebhookOperationConfigs() == null ? null + : JSONB.valueOf(Jsons.serialize(standardWorkspace.getWebhookOperationConfigs()))) .execute(); } }); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java index 898ec634a663..5d22a12c6029 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @@ -111,7 +111,8 @@ public WorkspaceServiceAccount getWorkspaceServiceAccountWithSecrets(final UUID public StandardWorkspace getWorkspaceWithSecrets(final UUID workspaceId, final boolean includeTombstone) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardWorkspace workspace = configRepository.getStandardWorkspaceNoSecrets(workspaceId, includeTombstone); - // TODO: hydrate any secrets once they're introduced. + final JsonNode webhookConfigs = secretsHydrator.hydrate(workspace.getWebhookOperationConfigs()); + workspace.withWebhookOperationConfigs(webhookConfigs); return workspace; } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java index b6f8a19423b7..176ec83aebd6 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @@ -83,7 +83,7 @@ public void writeSourceConnection(final SourceConnection source, final Connector source.getWorkspaceId(), previousSourceConnection, source.getConfiguration(), - connectorSpecification, + connectorSpecification.getConnectionSpecification(), source.getTombstone() == null || !source.getTombstone()); final SourceConnection partialSource = Jsons.clone(source).withConfiguration(partialConfig); @@ -107,7 +107,7 @@ public void writeDestinationConnection(final DestinationConnection destination, destination.getWorkspaceId(), previousDestinationConnection, destination.getConfiguration(), - connectorSpecification, + connectorSpecification.getConnectionSpecification(), destination.getTombstone() == null || !destination.getTombstone()); final DestinationConnection partialDestination = Jsons.clone(destination).withConfiguration(partialConfig); @@ -146,11 +146,11 @@ private JsonNode statefulSplitSecrets(final UUID workspaceId, final JsonNode ful private JsonNode statefulUpdateSecrets(final UUID workspaceId, final Optional oldConfig, final JsonNode fullConfig, - final ConnectorSpecification spec, + final JsonNode spec, final boolean validate) throws JsonValidationException { if (validate) { - validator.ensure(spec.getConnectionSpecification(), fullConfig); + validator.ensure(spec, fullConfig); } if (longLivedSecretPersistence.isEmpty()) { @@ -163,13 +163,13 @@ private JsonNode statefulUpdateSecrets(final UUID workspaceId, workspaceId, oldConfig.get(), fullConfig, - spec.getConnectionSpecification(), + spec, longLivedSecretPersistence.get()); } else { splitSecretConfig = SecretsHelpers.splitConfig( workspaceId, fullConfig, - spec.getConnectionSpecification()); + spec); } splitSecretConfig.getCoordinateToPayload().forEach(longLivedSecretPersistence.get()::write); return splitSecretConfig.getPartialConfig(); @@ -324,8 +324,35 @@ public Optional getOptionalWorkspaceServiceAccount(fina public void writeWorkspace(final StandardWorkspace workspace) throws JsonValidationException, IOException { - // TODO(msiega): split secrets once they're introduced. - configRepository.writeStandardWorkspaceNoSecrets(workspace); + // Get the schema for the webhook config so we can split out any secret fields. + final JsonNode webhookConfigSchema = Jsons.jsonNodeFromFile(ConfigSchema.WORKSPACE_WEBHOOK_OPERATION_CONFIGS.getConfigSchemaFile()); + // Check if there's an existing config, so we can re-use the secret coordinates. + final var previousWorkspace = getWorkspaceIfExists(workspace.getWorkspaceId(), false); + Optional previousWebhookConfigs = Optional.empty(); + if (previousWorkspace.isPresent() && previousWorkspace.get().getWebhookOperationConfigs() != null) { + previousWebhookConfigs = Optional.of(previousWorkspace.get().getWebhookOperationConfigs()); + } + // Split out the secrets from the webhook config. + final JsonNode partialConfig = workspace.getWebhookOperationConfigs() == null ? null + : statefulUpdateSecrets( + workspace.getWorkspaceId(), + previousWebhookConfigs, + workspace.getWebhookOperationConfigs(), + webhookConfigSchema, true); + final StandardWorkspace partialWorkspace = Jsons.clone(workspace); + if (partialConfig != null) { + partialWorkspace.withWebhookOperationConfigs(partialConfig); + } + configRepository.writeStandardWorkspaceNoSecrets(partialWorkspace); + } + + private Optional getWorkspaceIfExists(final UUID workspaceId, final boolean includeTombstone) { + try { + final StandardWorkspace existingWorkspace = configRepository.getStandardWorkspaceNoSecrets(workspaceId, includeTombstone); + return existingWorkspace == null ? Optional.empty() : Optional.of(existingWorkspace); + } catch (JsonValidationException | IOException | ConfigNotFoundException e) { + return Optional.empty(); + } } } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java index 85c5b86e5c8e..1ba17f029c73 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java @@ -11,6 +11,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -30,9 +31,13 @@ import io.airbyte.config.AirbyteConfig; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; +import io.airbyte.config.Geography; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.WebhookConfig; +import io.airbyte.config.WebhookOperationConfigs; import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.config.persistence.split_secrets.MemorySecretPersistence; import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; @@ -42,6 +47,7 @@ import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,6 +57,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -88,6 +95,11 @@ class SecretsRepositoryWriterTest { private static final String PASSWORD_PROPERTY_NAME = "password"; private static final String PASSWORD_FIELD_NAME = "_secret"; + private static final String TEST_EMAIL = "test-email"; + private static final String TEST_WORKSPACE_NAME = "test-workspace-name"; + private static final String TEST_WORKSPACE_SLUG = "test-workspace-slug"; + private static final String TEST_WEBHOOK_NAME = "test-webhook-name"; + private static final String TEST_AUTH_TOKEN = "test-auth-token"; private ConfigRepository configRepository; private MemorySecretPersistence longLivedSecretPersistence; @@ -433,4 +445,38 @@ void testWriteDifferentStagingConfiguration() throws JsonValidationException, Co Map.of(PASSWORD_FIELD_NAME, hmacSecretNewCoordinate.getFullCoordinate())))); } + @Test + @DisplayName("writeWorkspace should ensure that secret fields are replaced") + void testWriteWorkspaceSplitsAuthTokens() throws JsonValidationException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final SecretsRepositoryWriter secretsRepositoryWriter = + spy(new SecretsRepositoryWriter(configRepository, jsonSchemaValidator, Optional.of(secretPersistence), Optional.of(secretPersistence))); + final var webhookConfigs = new WebhookOperationConfigs().withWebhookConfigs(List.of( + new WebhookConfig() + .withName(TEST_WEBHOOK_NAME) + .withAuthToken(TEST_AUTH_TOKEN) + .withId(UUID.randomUUID()))); + final var workspace = new StandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withCustomerId(UUID.randomUUID()) + .withEmail(TEST_EMAIL) + .withName(TEST_WORKSPACE_NAME) + .withSlug(TEST_WORKSPACE_SLUG) + .withInitialSetupComplete(false) + .withDisplaySetupWizard(true) + .withNews(false) + .withAnonymousDataCollection(false) + .withSecurityUpdates(false) + .withTombstone(false) + .withNotifications(Collections.emptyList()) + .withDefaultGeography(Geography.AUTO) + // Serialize it to a string, then deserialize it to a JsonNode. + .withWebhookOperationConfigs(Jsons.jsonNode(webhookConfigs)); + secretsRepositoryWriter.writeWorkspace(workspace); + final var workspaceArgumentCaptor = ArgumentCaptor.forClass(StandardWorkspace.class); + verify(configRepository, times(1)).writeStandardWorkspaceNoSecrets(workspaceArgumentCaptor.capture()); + assertFalse(Jsons.serialize(workspaceArgumentCaptor.getValue().getWebhookOperationConfigs()).contains(TEST_AUTH_TOKEN)); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/WebhookOperationConfigsConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/WebhookOperationConfigsConverter.java new file mode 100644 index 000000000000..2317a19706d9 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/WebhookOperationConfigsConverter.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.converters; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.api.model.generated.WebhookConfigRead; +import io.airbyte.api.model.generated.WebhookConfigWrite; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.WebhookConfig; +import io.airbyte.config.WebhookOperationConfigs; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +public class WebhookOperationConfigsConverter { + + public static JsonNode toPersistenceWrite(List apiWebhookConfigs) { + if (apiWebhookConfigs == null) { + return Jsons.emptyObject(); + } + + final WebhookOperationConfigs configs = new WebhookOperationConfigs() + .withWebhookConfigs(apiWebhookConfigs.stream().map(WebhookOperationConfigsConverter::toPersistenceConfig).collect(Collectors.toList())); + + return Jsons.jsonNode(configs); + } + + public static List toApiReads(List persistenceConfig) { + if (persistenceConfig.isEmpty()) { + return Collections.emptyList(); + } + return persistenceConfig.stream().map(WebhookOperationConfigsConverter::toApiRead).collect(Collectors.toList()); + } + + private static WebhookConfig toPersistenceConfig(final WebhookConfigWrite input) { + return new WebhookConfig() + .withId(UUID.randomUUID()) + .withName(input.getName()) + .withAuthToken(input.getAuthToken()); + } + + private static WebhookConfigRead toApiRead(final WebhookConfig persistenceConfig) { + final var read = new WebhookConfigRead(); + read.setId(persistenceConfig.getId()); + read.setName(persistenceConfig.getName()); + return read; + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java index 71ece0d59250..7e949a89d201 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java @@ -24,18 +24,22 @@ import io.airbyte.api.model.generated.WorkspaceUpdate; import io.airbyte.api.model.generated.WorkspaceUpdateName; import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.json.Jsons; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.WebhookOperationConfigs; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.notification.NotificationClient; import io.airbyte.server.converters.NotificationConverter; +import io.airbyte.server.converters.WebhookOperationConfigsConverter; import io.airbyte.server.errors.IdNotFoundKnownException; import io.airbyte.server.errors.InternalServerKnownException; import io.airbyte.server.errors.ValueConflictKnownException; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -103,7 +107,8 @@ public WorkspaceRead createWorkspace(final WorkspaceCreate workspaceCreate) .withDisplaySetupWizard(displaySetupWizard != null ? displaySetupWizard : false) .withTombstone(false) .withNotifications(NotificationConverter.toConfigList(workspaceCreate.getNotifications())) - .withDefaultGeography(defaultGeography); + .withDefaultGeography(defaultGeography) + .withWebhookOperationConfigs(WebhookOperationConfigsConverter.toPersistenceWrite(workspaceCreate.getWebhookConfigs())); if (!Strings.isNullOrEmpty(email)) { workspace.withEmail(email); @@ -250,7 +255,7 @@ private String generateUniqueSlug(final String workspaceName) throws JsonValidat } private static WorkspaceRead buildWorkspaceRead(final StandardWorkspace workspace) { - return new WorkspaceRead() + final WorkspaceRead result = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) .customerId(workspace.getCustomerId()) .email(workspace.getEmail()) @@ -263,6 +268,15 @@ private static WorkspaceRead buildWorkspaceRead(final StandardWorkspace workspac .securityUpdates(workspace.getSecurityUpdates()) .notifications(NotificationConverter.toApiList(workspace.getNotifications())) .defaultGeography(Enums.convertTo(workspace.getDefaultGeography(), Geography.class)); + // Add read-only webhook configs. + final Optional persistedConfigs = Jsons.tryObject( + workspace.getWebhookOperationConfigs(), + WebhookOperationConfigs.class); + if (persistedConfigs.isPresent()) { + result.setWebhookConfigs(WebhookOperationConfigsConverter.toApiReads( + persistedConfigs.get().getWebhookConfigs())); + } + return result; } private void validateWorkspacePatch(final StandardWorkspace persistedWorkspace, final WorkspaceUpdate workspacePatch) { diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java index d8a431823e50..8220509c9eaa 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java @@ -54,6 +54,7 @@ class WorkspacesHandlerTest { public static final String FAILURE_NOTIFICATION_WEBHOOK = "http://airbyte.notifications/failure"; + public static final String NEW_WORKSPACE = "new workspace"; private ConfigRepository configRepository; private SecretsRepositoryWriter secretsRepositoryWriter; private ConnectionsHandler connectionsHandler; @@ -81,6 +82,7 @@ void setUp() { destinationHandler = mock(DestinationHandler.class); sourceHandler = mock(SourceHandler.class); uuidSupplier = mock(Supplier.class); + workspace = generateWorkspace(); workspacesHandler = new WorkspacesHandler(configRepository, secretsRepositoryWriter, connectionsHandler, destinationHandler, sourceHandler, uuidSupplier); @@ -118,8 +120,8 @@ private io.airbyte.api.model.generated.Notification generateApiNotification() { } @Test - void testCreateWorkspace() throws JsonValidationException, IOException { - when(configRepository.listStandardWorkspaces(false)).thenReturn(Collections.singletonList(workspace)); + void testCreateWorkspace() throws JsonValidationException, IOException, ConfigNotFoundException { + when(configRepository.getStandardWorkspaceNoSecrets(any(), eq(false))).thenReturn(workspace); final UUID uuid = UUID.randomUUID(); when(uuidSupplier.get()).thenReturn(uuid); @@ -127,7 +129,7 @@ void testCreateWorkspace() throws JsonValidationException, IOException { configRepository.writeStandardWorkspaceNoSecrets(workspace); final WorkspaceCreate workspaceCreate = new WorkspaceCreate() - .name("new workspace") + .name(NEW_WORKSPACE) .email(TEST_EMAIL) .news(false) .anonymousDataCollection(false) @@ -140,7 +142,7 @@ void testCreateWorkspace() throws JsonValidationException, IOException { .workspaceId(uuid) .customerId(uuid) .email(TEST_EMAIL) - .name("new workspace") + .name(NEW_WORKSPACE) .slug("new-workspace") .initialSetupComplete(false) .displaySetupWizard(false) @@ -148,17 +150,19 @@ void testCreateWorkspace() throws JsonValidationException, IOException { .anonymousDataCollection(false) .securityUpdates(false) .notifications(List.of(generateApiNotification())) - .defaultGeography(GEOGRAPHY_US); + .defaultGeography(GEOGRAPHY_US) + .webhookConfigs(Collections.emptyList()); assertEquals(expectedRead, actualRead); } @Test - void testCreateWorkspaceDuplicateSlug() throws JsonValidationException, IOException { + void testCreateWorkspaceDuplicateSlug() throws JsonValidationException, IOException, ConfigNotFoundException { when(configRepository.getWorkspaceBySlugOptional(any(String.class), eq(true))) .thenReturn(Optional.of(workspace)) .thenReturn(Optional.of(workspace)) .thenReturn(Optional.empty()); + when(configRepository.getStandardWorkspaceNoSecrets(any(), eq(false))).thenReturn(workspace); final UUID uuid = UUID.randomUUID(); when(uuidSupplier.get()).thenReturn(uuid); @@ -186,7 +190,8 @@ void testCreateWorkspaceDuplicateSlug() throws JsonValidationException, IOExcept .anonymousDataCollection(false) .securityUpdates(false) .notifications(Collections.emptyList()) - .defaultGeography(GEOGRAPHY_AUTO); + .defaultGeography(GEOGRAPHY_AUTO) + .webhookConfigs(Collections.emptyList()); assertTrue(actualRead.getSlug().startsWith(workspace.getSlug())); assertNotEquals(workspace.getSlug(), actualRead.getSlug()); @@ -461,4 +466,42 @@ void testSetFeedbackDone() throws JsonValidationException, ConfigNotFoundExcepti verify(configRepository).setFeedback(workspaceGiveFeedback.getWorkspaceId()); } + @Test + void testWorkspaceIsWrittenThroughSecretsWriter() throws JsonValidationException, IOException { + secretsRepositoryWriter = mock(SecretsRepositoryWriter.class); + workspacesHandler = new WorkspacesHandler(configRepository, secretsRepositoryWriter, connectionsHandler, + destinationHandler, sourceHandler, uuidSupplier); + + final UUID uuid = UUID.randomUUID(); + when(uuidSupplier.get()).thenReturn(uuid); + + final WorkspaceCreate workspaceCreate = new WorkspaceCreate() + .name(NEW_WORKSPACE) + .email(TEST_EMAIL) + .news(false) + .anonymousDataCollection(false) + .securityUpdates(false) + .notifications(List.of(generateApiNotification())) + .defaultGeography(GEOGRAPHY_US); + + final WorkspaceRead actualRead = workspacesHandler.createWorkspace(workspaceCreate); + final WorkspaceRead expectedRead = new WorkspaceRead() + .workspaceId(uuid) + .customerId(uuid) + .email(TEST_EMAIL) + .name(NEW_WORKSPACE) + .slug("new-workspace") + .initialSetupComplete(false) + .displaySetupWizard(false) + .news(false) + .anonymousDataCollection(false) + .securityUpdates(false) + .notifications(List.of(generateApiNotification())) + .defaultGeography(GEOGRAPHY_US) + .webhookConfigs(Collections.emptyList()); + + assertEquals(expectedRead, actualRead); + verify(secretsRepositoryWriter, times(1)).writeWorkspace(any()); + } + } diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 2801d6cbc913..ca18b820a0da 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -9142,6 +9142,13 @@

Example data

Content-Type: application/json
{
   "news" : true,
+  "webhookConfigs" : [ {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ],
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -9271,6 +9278,13 @@ 

Example data

Content-Type: application/json
{
   "news" : true,
+  "webhookConfigs" : [ {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ],
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -9355,6 +9369,13 @@ 

Example data

Content-Type: application/json
{
   "news" : true,
+  "webhookConfigs" : [ {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ],
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -9428,6 +9449,13 @@ 

Example data

{
   "workspaces" : [ {
     "news" : true,
+    "webhookConfigs" : [ {
+      "name" : "name",
+      "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    }, {
+      "name" : "name",
+      "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    } ],
     "displaySetupWizard" : true,
     "initialSetupComplete" : true,
     "anonymousDataCollection" : true,
@@ -9456,6 +9484,13 @@ 

Example data

"workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, { "news" : true, + "webhookConfigs" : [ { + "name" : "name", + "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" + }, { + "name" : "name", + "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" + } ], "displaySetupWizard" : true, "initialSetupComplete" : true, "anonymousDataCollection" : true, @@ -9535,6 +9570,13 @@

Example data

Content-Type: application/json
{
   "news" : true,
+  "webhookConfigs" : [ {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ],
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -9664,6 +9706,13 @@ 

Example data

Content-Type: application/json
{
   "news" : true,
+  "webhookConfigs" : [ {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "name" : "name",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ],
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -9875,6 +9924,8 @@ 

Table of Contents

  • WebBackendOperationCreateOrUpdate -
  • WebBackendWorkspaceState -
  • WebBackendWorkspaceStateResult -
  • +
  • WebhookConfigRead -
  • +
  • WebhookConfigWrite -
  • WorkspaceCreate -
  • WorkspaceGiveFeedback -
  • WorkspaceIdRequestBody -
  • @@ -11374,6 +11425,23 @@

    WebBackendWorkspaceStateResul
    hasDestinations
    +
    +

    WebhookConfigRead - Up

    +
    the readable info for a webhook config; omits sensitive info e.g. auth token
    +
    +
    id
    UUID format: uuid
    +
    name (optional)
    String human-readable name e.g. for display in UI
    +
    +
    +
    +

    WebhookConfigWrite - Up

    +
    +
    +
    name (optional)
    String human readable name for this webhook e.g. for UI display.
    +
    authToken (optional)
    String an auth token, to be passed as the value for an HTTP Authorization header.
    +
    validationUrl (optional)
    String if supplied, the webhook config will be validated by checking that this URL returns a 2xx response.
    +
    +
    displaySetupWizard (optional)
    defaultGeography (optional)
    +
    webhookConfigs (optional)
    feedbackDone (optional)
    defaultGeography (optional)
    +
    webhookConfigs (optional)