diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index 10a36e9fb2ae..63356bf71fbc 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -1,7 +1,9 @@ import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { - id "org.openapi.generator" version "5.2.0" + // 6.1.0-SNAPSHOT contains this fix: https://github.com/OpenAPITools/openapi-generator/issues/13025 + // TODO update to v6.1.0 when it officially releases (due end of August 2022) so that we don't need to depend on a SNAPSHOT version anymore + id "org.openapi.generator" version "6.1.0-SNAPSHOT" id "java-library" } @@ -22,7 +24,7 @@ task generateApiServerLegacy(type: GenerateTask) { invokerPackage = "io.airbyte.api.invoker.generated" modelPackage = "io.airbyte.api.model.generated" - importMappings = [ + schemaMappings = [ 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', @@ -64,7 +66,7 @@ task generateApiServer(type: GenerateTask) { invokerPackage = "io.airbyte.api.invoker.generated" modelPackage = "io.airbyte.api.model.generated" - importMappings = [ + schemaMappings = [ 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', @@ -111,7 +113,7 @@ task generateApiClient(type: GenerateTask) { invokerPackage = "io.airbyte.api.client.invoker.generated" modelPackage = "io.airbyte.api.client.model.generated" - importMappings = [ + schemaMappings = [ 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', @@ -145,7 +147,7 @@ task generateApiDocs(type: GenerateTask) { invokerPackage = "io.airbyte.api.client.invoker.generated" modelPackage = "io.airbyte.api.client.model.generated" - importMappings = [ + schemaMappings = [ 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java index 688e2cb9878e..93710d06bf39 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java @@ -43,7 +43,6 @@ import io.airbyte.workers.helper.ProtocolConverters; import java.io.IOException; import java.nio.file.Path; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -170,7 +169,11 @@ private static List getAttemptStreamStats(final Attempt atte .map(JobOutput::getSync) .map(StandardSyncOutput::getStandardSyncSummary) .map(StandardSyncSummary::getStreamStats) - .orElse(Collections.emptyList()); + .orElse(null); + + if (streamStats == null) { + return null; + } return streamStats.stream() .map(streamStat -> new AttemptStreamStats() diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/NotificationConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/NotificationConverter.java index 93c91227d440..c24346a123f5 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/NotificationConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/NotificationConverter.java @@ -5,12 +5,16 @@ package io.airbyte.server.converters; import io.airbyte.commons.enums.Enums; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; public class NotificationConverter { public static List toConfigList(final List notifications) { + if (notifications == null) { + return Collections.emptyList(); + } return notifications.stream().map(NotificationConverter::toConfig).collect(Collectors.toList()); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java index 4e965afb534a..cfda5cdf69cb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java @@ -111,10 +111,12 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) // Set this as default name if connectionCreate doesn't have it final String defaultName = sourceConnection.getName() + " <> " + destinationConnection.getName(); + final List operationIds = connectionCreate.getOperationIds() != null ? connectionCreate.getOperationIds() : Collections.emptyList(); + ConnectionHelper.validateWorkspace(workspaceHelper, connectionCreate.getSourceId(), connectionCreate.getDestinationId(), - new HashSet<>(connectionCreate.getOperationIds())); + new HashSet<>(operationIds)); final UUID connectionId = uuidGenerator.get(); @@ -127,7 +129,7 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) .withPrefix(connectionCreate.getPrefix()) .withSourceId(connectionCreate.getSourceId()) .withDestinationId(connectionCreate.getDestinationId()) - .withOperationIds(connectionCreate.getOperationIds()) + .withOperationIds(operationIds) .withStatus(ApiPojoConverters.toPersistenceStatus(connectionCreate.getStatus())) .withSourceCatalogId(connectionCreate.getSourceCatalogId()); if (connectionCreate.getResourceRequirements() != null) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index a017ad9e9740..2c5fcf56a7e6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -423,6 +423,9 @@ private void verifyManualOperationResult(final ManualOperationResult manualOpera private List createOperations(final WebBackendConnectionCreate webBackendConnectionCreate) throws JsonValidationException, ConfigNotFoundException, IOException { + if (webBackendConnectionCreate.getOperations() == null) { + return Collections.emptyList(); + } final List operationIds = new ArrayList<>(); for (final var operationCreate : webBackendConnectionCreate.getOperations()) { operationIds.add(operationsHandler.createOperation(operationCreate).getOperationId()); @@ -434,10 +437,17 @@ private List updateOperations(final WebBackendConnectionUpdate webBackendC throws JsonValidationException, ConfigNotFoundException, IOException { final ConnectionRead connectionRead = connectionsHandler .getConnection(webBackendConnectionUpdate.getConnectionId()); - final List originalOperationIds = new ArrayList<>(connectionRead.getOperationIds()); + + // wrap operationIds in a new ArrayList so that it is modifiable below, when calling .removeAll + final List originalOperationIds = + connectionRead.getOperationIds() == null ? new ArrayList<>() : new ArrayList<>(connectionRead.getOperationIds()); + + final List updatedOperations = + webBackendConnectionUpdate.getOperations() == null ? new ArrayList<>() : webBackendConnectionUpdate.getOperations(); + final List operationIds = new ArrayList<>(); - for (final var operationCreateOrUpdate : webBackendConnectionUpdate.getOperations()) { + for (final var operationCreateOrUpdate : updatedOperations) { if (operationCreateOrUpdate.getOperationId() == null || !originalOperationIds.contains(operationCreateOrUpdate.getOperationId())) { final OperationCreate operationCreate = toOperationCreate(operationCreateOrUpdate); operationIds.add(operationsHandler.createOperation(operationCreate).getOperationId()); @@ -446,6 +456,7 @@ private List updateOperations(final WebBackendConnectionUpdate webBackendC operationIds.add(operationsHandler.updateOperation(operationUpdate).getOperationId()); } } + originalOperationIds.removeAll(operationIds); operationsHandler.deleteOperationsForConnection(connectionRead.getConnectionId(), originalOperationIds); return operationIds; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/CatalogConverter.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/CatalogConverter.java index ad2ae7e4e194..174fec49e50e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/CatalogConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/CatalogConverter.java @@ -9,7 +9,9 @@ import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.text.Names; +import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -35,7 +37,7 @@ private static io.airbyte.protocol.models.AirbyteStream toProtocol(final io.airb .withSupportedSyncModes(Enums.convertListTo(stream.getSupportedSyncModes(), io.airbyte.protocol.models.SyncMode.class)) .withSourceDefinedCursor(stream.getSourceDefinedCursor()) .withDefaultCursorField(stream.getDefaultCursorField()) - .withSourceDefinedPrimaryKey(stream.getSourceDefinedPrimaryKey()) + .withSourceDefinedPrimaryKey(Optional.ofNullable(stream.getSourceDefinedPrimaryKey()).orElse(Collections.emptyList())) .withNamespace(stream.getNamespace()); } @@ -119,7 +121,7 @@ public static io.airbyte.protocol.models.ConfiguredAirbyteCatalog toProtocol(fin .withCursorField(s.getConfig().getCursorField()) .withDestinationSyncMode(Enums.convertTo(s.getConfig().getDestinationSyncMode(), io.airbyte.protocol.models.DestinationSyncMode.class)) - .withPrimaryKey(s.getConfig().getPrimaryKey())) + .withPrimaryKey(Optional.ofNullable(s.getConfig().getPrimaryKey()).orElse(Collections.emptyList()))) .collect(Collectors.toList()); return new io.airbyte.protocol.models.ConfiguredAirbyteCatalog() .withStreams(streams); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index ae54963864a3..524fc347092c 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -124,7 +124,8 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() .destinationDefinitionId(destination2.getDestinationDefinitionId()) @@ -137,7 +138,8 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionReadList actualDestinationDefinitionReadList = destinationDefinitionsHandler.listDestinationDefinitions(); @@ -165,7 +167,8 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() .destinationDefinitionId(destination2.getDestinationDefinitionId()) @@ -178,7 +181,8 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionReadList actualDestinationDefinitionReadList = destinationDefinitionsHandler .listDestinationDefinitionsForWorkspace(new WorkspaceIdRequestBody().workspaceId(workspaceId)); @@ -209,7 +213,8 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() .destinationDefinitionId(destinationDefinition2.getDestinationDefinitionId()) @@ -222,7 +227,8 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition2.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destinationDefinition2.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final PrivateDestinationDefinitionRead expectedDestinationDefinitionOptInRead1 = new PrivateDestinationDefinitionRead().destinationDefinition(expectedDestinationDefinitionRead1).granted(false); @@ -256,7 +262,8 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody = new DestinationDefinitionIdRequestBody() .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()); @@ -300,7 +307,8 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId = new DestinationDefinitionIdWithWorkspaceId() .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()) @@ -331,7 +339,8 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J .icon(destination.getIcon()) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead expectedRead = new DestinationDefinitionRead() .name(destination.getName()) @@ -343,7 +352,8 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead actualRead = destinationDefinitionsHandler.createPrivateDestinationDefinition(create); @@ -373,7 +383,8 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept .icon(destination.getIcon()) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final CustomDestinationDefinitionCreate customCreate = new CustomDestinationDefinitionCreate() .destinationDefinition(create) @@ -389,7 +400,8 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead actualRead = destinationDefinitionsHandler.createCustomDestinationDefinition(customCreate); @@ -471,7 +483,8 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException, .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final PrivateDestinationDefinitionRead expectedPrivateDestinationDefinitionRead = new PrivateDestinationDefinitionRead().destinationDefinition(expectedDestinationDefinitionRead).granted(true); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java index 848ec26bf31f..c154815be027 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java @@ -123,7 +123,8 @@ void testListSourceDefinitions() throws JsonValidationException, IOException, UR .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionRead expectedSourceDefinitionRead2 = new SourceDefinitionRead() .sourceDefinitionId(sourceDefinition2.getSourceDefinitionId()) @@ -136,7 +137,8 @@ void testListSourceDefinitions() throws JsonValidationException, IOException, UR .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionReadList actualSourceDefinitionReadList = sourceDefinitionsHandler.listSourceDefinitions(); @@ -164,7 +166,8 @@ void testListSourceDefinitionsForWorkspace() throws IOException, URISyntaxExcept .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionRead expectedSourceDefinitionRead2 = new SourceDefinitionRead() .sourceDefinitionId(sourceDefinition2.getSourceDefinitionId()) @@ -177,7 +180,8 @@ void testListSourceDefinitionsForWorkspace() throws IOException, URISyntaxExcept .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionReadList actualSourceDefinitionReadList = sourceDefinitionsHandler.listSourceDefinitionsForWorkspace(new WorkspaceIdRequestBody().workspaceId(workspaceId)); @@ -208,7 +212,8 @@ void testListPrivateSourceDefinitions() throws IOException, URISyntaxException { .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionRead expectedSourceDefinitionRead2 = new SourceDefinitionRead() .sourceDefinitionId(sourceDefinition2.getSourceDefinitionId()) @@ -221,7 +226,8 @@ void testListPrivateSourceDefinitions() throws IOException, URISyntaxException { .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final PrivateSourceDefinitionRead expectedSourceDefinitionOptInRead1 = new PrivateSourceDefinitionRead().sourceDefinition(expectedSourceDefinitionRead1).granted(false); @@ -254,7 +260,8 @@ void testGetSourceDefinition() throws JsonValidationException, ConfigNotFoundExc .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody = new SourceDefinitionIdRequestBody().sourceDefinitionId(sourceDefinition.getSourceDefinitionId()); @@ -296,7 +303,8 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId = new SourceDefinitionIdWithWorkspaceId() .sourceDefinitionId(sourceDefinition.getSourceDefinitionId()) @@ -327,7 +335,8 @@ void testCreateSourceDefinition() throws URISyntaxException, IOException, JsonVa .icon(sourceDefinition.getIcon()) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionRead expectedRead = new SourceDefinitionRead() .name(sourceDefinition.getName()) @@ -339,7 +348,8 @@ void testCreateSourceDefinition() throws URISyntaxException, IOException, JsonVa .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionRead actualRead = sourceDefinitionsHandler.createPrivateSourceDefinition(create); @@ -368,7 +378,8 @@ void testCreateCustomSourceDefinition() throws URISyntaxException, IOException, .icon(sourceDefinition.getIcon()) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final CustomSourceDefinitionCreate customCreate = new CustomSourceDefinitionCreate() .sourceDefinition(create) @@ -384,7 +395,8 @@ void testCreateCustomSourceDefinition() throws URISyntaxException, IOException, .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final SourceDefinitionRead actualRead = sourceDefinitionsHandler.createCustomSourceDefinition(customCreate); @@ -464,7 +476,8 @@ void testGrantSourceDefinitionToWorkspace() throws JsonValidationException, Conf .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest())) + .jobSpecific(Collections.emptyList())); final PrivateSourceDefinitionRead expectedPrivateSourceDefinitionRead = new PrivateSourceDefinitionRead().sourceDefinition(expectedSourceDefinitionRead).granted(true); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index b2254e505942..758610d07b4a 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -483,8 +483,8 @@ void testToConnectionUpdate() throws IOException { @Test void testForConnectionCreateCompleteness() { final Set handledMethods = - Set.of("name", "namespaceDefinition", "namespaceFormat", "prefix", "sourceId", "destinationId", "operationIds", "syncCatalog", "schedule", - "scheduleType", "scheduleData", "status", "resourceRequirements", "sourceCatalogId"); + Set.of("name", "namespaceDefinition", "namespaceFormat", "prefix", "sourceId", "destinationId", "operationIds", "addOperationIdsItem", + "removeOperationIdsItem", "syncCatalog", "schedule", "scheduleType", "scheduleData", "status", "resourceRequirements", "sourceCatalogId"); final Set methods = Arrays.stream(ConnectionCreate.class.getMethods()) .filter(method -> method.getReturnType() == ConnectionCreate.class) @@ -505,7 +505,7 @@ void testForConnectionCreateCompleteness() { void testForConnectionUpdateCompleteness() { final Set handledMethods = Set.of("schedule", "connectionId", "syncCatalog", "namespaceDefinition", "namespaceFormat", "prefix", "status", "operationIds", - "resourceRequirements", "name", "sourceCatalogId", "scheduleType", "scheduleData"); + "addOperationIdsItem", "removeOperationIdsItem", "resourceRequirements", "name", "sourceCatalogId", "scheduleType", "scheduleData"); final Set methods = Arrays.stream(ConnectionUpdate.class.getMethods()) .filter(method -> method.getReturnType() == ConnectionUpdate.class) diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java index cbd076b72618..0fe25ae36bdb 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java @@ -253,6 +253,7 @@ private static AirbyteStream generateBasicApiStream(final String nameSuffix) { .jsonSchema(generateBasicJsonSchema()) .defaultCursorField(Lists.newArrayList(FIELD_NAME)) .sourceDefinedCursor(false) + .sourceDefinedPrimaryKey(Collections.emptyList()) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); } diff --git a/build.gradle b/build.gradle index 1fedfe594eab..e5b387fde5d2 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,9 @@ import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.github.spotbugs.snom.SpotBugsTask +// The buildscript block defines dependencies in order for .gradle file evaluation. +// This is separate from application dependencies. +// See https://stackoverflow.com/questions/17773817/purpose-of-buildscript-block-in-gradle. buildscript { repositories { maven { @@ -9,6 +12,14 @@ buildscript { } dependencies { classpath 'com.bmuschko:gradle-docker-plugin:7.2.0' + // 6.x version of OpenApi generator is only compatible with jackson-core 2.13.x onwards. + // This conflicts with the jackson depencneis the bmuschko plugin is pulling in. + // Since api generation is only used in the airbyte-api module and the base gradle files + // are loaded in first, Gradle is not able to intelligently resolve this before loading in + // the bmuschko plugin and thus placing an older jackson version on the class path. + // The alternative is to import the openapi plugin for all modules. + // This might need to be updated when we change openapi plugin versions. + classpath 'com.fasterxml.jackson.core:jackson-core:2.13.0' } } diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 6f4abe5a6650..0de7ded6d3e0 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -5395,7 +5395,7 @@

Request body

Return type

- map[String, Object] + map[String, oas_any_type_not_mapped]
@@ -5411,7 +5411,7 @@

Produces

Responses

200

Successful operation - map[String, Object] +

404

Object with given id was not found. NotFoundKnownExceptionInfo @@ -5448,7 +5448,7 @@

Request body

Return type

- map[String, Object] + map[String, oas_any_type_not_mapped]
@@ -5464,7 +5464,7 @@

Produces

Responses

200

Successful operation - map[String, Object] +

404

Object with given id was not found. NotFoundKnownExceptionInfo @@ -11100,7 +11100,7 @@

CompleteDestinationOAuthRequ
destinationDefinitionId
UUID format: uuid
workspaceId
UUID format: uuid
redirectUrl (optional)
String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
-
queryParams (optional)
map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
+
queryParams (optional)
map[String, oas_any_type_not_mapped] The query parameters present in the redirect URL after a user granted consent e.g auth code
oAuthInputConfiguration (optional)
@@ -11111,7 +11111,7 @@

CompleteSourceOauthRequest
sourceDefinitionId
UUID format: uuid
workspaceId
UUID format: uuid
redirectUrl (optional)
String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
-
queryParams (optional)
map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
+
queryParams (optional)
map[String, oas_any_type_not_mapped] The query parameters present in the redirect URL after a user granted consent e.g auth code
oAuthInputConfiguration (optional)
@@ -11970,7 +11970,7 @@

SetInstance
diff --git a/settings.gradle b/settings.gradle index 20fc7bc37c0c..494a5f42ffa5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,22 @@ +pluginManagement { + repositories { + gradlePluginPortal() + maven { + url 'https://oss.sonatype.org/content/repositories/snapshots' + } + } + resolutionStrategy { + eachPlugin { + // We're using the 6.1.0-SNAPSHOT version of openapi-generator which contains a fix for generating nullable arrays (https://github.com/OpenAPITools/openapi-generator/issues/13025) + // The snapshot version isn't available in the main Gradle Plugin Portal, so we added the Sonatype snapshot repository above. + // The useModule command below allows us to map from the plugin id, `org.openapi.generator`, to the underlying module (https://oss.sonatype.org/content/repositories/snapshots/org/openapitools/openapi-generator-gradle-plugin/6.1.0-SNAPSHOT/_ + if (requested.id.id == 'org.openapi.generator') { + useModule "org.openapitools:openapi-generator-gradle-plugin:${requested.version}" + } + } + } +} + // Configure the gradle enterprise plugin to enable build scans. Enabling the plugin at the top of the settings file allows the build scan to record // as much information as possible. plugins {