Upgrade OpenAPI Generator to 6.1.0-SNAPSHOT (#16026)

airbytehq · Aug 27, 2022 · 6a06c5b · 6a06c5b
1 parent f3f895d
commit 6a06c5b
Show file tree

Hide file tree

Showing 13 changed files with 131 additions and 50 deletions.
diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle
@@ -1,7 +1,9 @@
 import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
 
 plugins {
-    id "org.openapi.generator" version "5.2.0"
+    // 6.1.0-SNAPSHOT contains this fix: https://github.com/OpenAPITools/openapi-generator/issues/13025
+    // TODO update to v6.1.0 when it officially releases (due end of August 2022) so that we don't need to depend on a SNAPSHOT version anymore
+    id "org.openapi.generator" version "6.1.0-SNAPSHOT"
     id "java-library"
 }
 
@@ -22,7 +24,7 @@ task generateApiServerLegacy(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.invoker.generated"
     modelPackage = "io.airbyte.api.model.generated"
 
-    importMappings = [
+    schemaMappings = [
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
@@ -64,7 +66,7 @@ task generateApiServer(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.invoker.generated"
     modelPackage = "io.airbyte.api.model.generated"
 
-    importMappings = [
+    schemaMappings = [
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
@@ -111,7 +113,7 @@ task generateApiClient(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.client.invoker.generated"
     modelPackage = "io.airbyte.api.client.model.generated"
 
-    importMappings = [
+    schemaMappings = [
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
@@ -145,7 +147,7 @@ task generateApiDocs(type: GenerateTask) {
     invokerPackage = "io.airbyte.api.client.invoker.generated"
     modelPackage = "io.airbyte.api.client.model.generated"
 
-    importMappings = [
+    schemaMappings = [
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
             'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',

diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java
@@ -43,7 +43,6 @@
 import io.airbyte.workers.helper.ProtocolConverters;
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -170,7 +169,11 @@ private static List<AttemptStreamStats> getAttemptStreamStats(final Attempt atte
         .map(JobOutput::getSync)
         .map(StandardSyncOutput::getStandardSyncSummary)
         .map(StandardSyncSummary::getStreamStats)
-        .orElse(Collections.emptyList());
+        .orElse(null);
+
+    if (streamStats == null) {
+      return null;
+    }
 
     return streamStats.stream()
         .map(streamStat -> new AttemptStreamStats()

diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/NotificationConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/NotificationConverter.java
@@ -5,12 +5,16 @@
 package io.airbyte.server.converters;
 
 import io.airbyte.commons.enums.Enums;
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
 public class NotificationConverter {
 
   public static List<io.airbyte.config.Notification> toConfigList(final List<io.airbyte.api.model.generated.Notification> notifications) {
+    if (notifications == null) {
+      return Collections.emptyList();
+    }
     return notifications.stream().map(NotificationConverter::toConfig).collect(Collectors.toList());
   }
 

diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java
@@ -111,10 +111,12 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate)
     // Set this as default name if connectionCreate doesn't have it
     final String defaultName = sourceConnection.getName() + " <> " + destinationConnection.getName();
 
+    final List<UUID> operationIds = connectionCreate.getOperationIds() != null ? connectionCreate.getOperationIds() : Collections.emptyList();
+
     ConnectionHelper.validateWorkspace(workspaceHelper,
         connectionCreate.getSourceId(),
         connectionCreate.getDestinationId(),
-        new HashSet<>(connectionCreate.getOperationIds()));
+        new HashSet<>(operationIds));
 
     final UUID connectionId = uuidGenerator.get();
 
@@ -127,7 +129,7 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate)
         .withPrefix(connectionCreate.getPrefix())
         .withSourceId(connectionCreate.getSourceId())
         .withDestinationId(connectionCreate.getDestinationId())
-        .withOperationIds(connectionCreate.getOperationIds())
+        .withOperationIds(operationIds)
         .withStatus(ApiPojoConverters.toPersistenceStatus(connectionCreate.getStatus()))
         .withSourceCatalogId(connectionCreate.getSourceCatalogId());
     if (connectionCreate.getResourceRequirements() != null) {

diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java
@@ -423,6 +423,9 @@ private void verifyManualOperationResult(final ManualOperationResult manualOpera
 
   private List<UUID> createOperations(final WebBackendConnectionCreate webBackendConnectionCreate)
       throws JsonValidationException, ConfigNotFoundException, IOException {
+    if (webBackendConnectionCreate.getOperations() == null) {
+      return Collections.emptyList();
+    }
     final List<UUID> operationIds = new ArrayList<>();
     for (final var operationCreate : webBackendConnectionCreate.getOperations()) {
       operationIds.add(operationsHandler.createOperation(operationCreate).getOperationId());
@@ -434,10 +437,17 @@ private List<UUID> updateOperations(final WebBackendConnectionUpdate webBackendC
       throws JsonValidationException, ConfigNotFoundException, IOException {
     final ConnectionRead connectionRead = connectionsHandler
         .getConnection(webBackendConnectionUpdate.getConnectionId());
-    final List<UUID> originalOperationIds = new ArrayList<>(connectionRead.getOperationIds());
+
+    // wrap operationIds in a new ArrayList so that it is modifiable below, when calling .removeAll
+    final List<UUID> originalOperationIds =
+        connectionRead.getOperationIds() == null ? new ArrayList<>() : new ArrayList<>(connectionRead.getOperationIds());
+
+    final List<WebBackendOperationCreateOrUpdate> updatedOperations =
+        webBackendConnectionUpdate.getOperations() == null ? new ArrayList<>() : webBackendConnectionUpdate.getOperations();
+
     final List<UUID> operationIds = new ArrayList<>();
 
-    for (final var operationCreateOrUpdate : webBackendConnectionUpdate.getOperations()) {
+    for (final var operationCreateOrUpdate : updatedOperations) {
       if (operationCreateOrUpdate.getOperationId() == null || !originalOperationIds.contains(operationCreateOrUpdate.getOperationId())) {
         final OperationCreate operationCreate = toOperationCreate(operationCreateOrUpdate);
         operationIds.add(operationsHandler.createOperation(operationCreate).getOperationId());
@@ -446,6 +456,7 @@ private List<UUID> updateOperations(final WebBackendConnectionUpdate webBackendC
         operationIds.add(operationsHandler.updateOperation(operationUpdate).getOperationId());
       }
     }
+
     originalOperationIds.removeAll(operationIds);
     operationsHandler.deleteOperationsForConnection(connectionRead.getConnectionId(), originalOperationIds);
     return operationIds;

diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/CatalogConverter.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/CatalogConverter.java
@@ -9,7 +9,9 @@
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.text.Names;
+import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
@@ -35,7 +37,7 @@ private static io.airbyte.protocol.models.AirbyteStream toProtocol(final io.airb
         .withSupportedSyncModes(Enums.convertListTo(stream.getSupportedSyncModes(), io.airbyte.protocol.models.SyncMode.class))
         .withSourceDefinedCursor(stream.getSourceDefinedCursor())
         .withDefaultCursorField(stream.getDefaultCursorField())
-        .withSourceDefinedPrimaryKey(stream.getSourceDefinedPrimaryKey())
+        .withSourceDefinedPrimaryKey(Optional.ofNullable(stream.getSourceDefinedPrimaryKey()).orElse(Collections.emptyList()))
         .withNamespace(stream.getNamespace());
   }
 
@@ -119,7 +121,7 @@ public static io.airbyte.protocol.models.ConfiguredAirbyteCatalog toProtocol(fin
             .withCursorField(s.getConfig().getCursorField())
             .withDestinationSyncMode(Enums.convertTo(s.getConfig().getDestinationSyncMode(),
                 io.airbyte.protocol.models.DestinationSyncMode.class))
-            .withPrimaryKey(s.getConfig().getPrimaryKey()))
+            .withPrimaryKey(Optional.ofNullable(s.getConfig().getPrimaryKey()).orElse(Collections.emptyList())))
         .collect(Collectors.toList());
     return new io.airbyte.protocol.models.ConfiguredAirbyteCatalog()
         .withStreams(streams);

diff --git a/...te-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/...te-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java
@@ -124,7 +124,8 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead()
         .destinationDefinitionId(destination2.getDestinationDefinitionId())
@@ -137,7 +138,8 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionReadList actualDestinationDefinitionReadList = destinationDefinitionsHandler.listDestinationDefinitions();
 
@@ -165,7 +167,8 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead()
         .destinationDefinitionId(destination2.getDestinationDefinitionId())
@@ -178,7 +181,8 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionReadList actualDestinationDefinitionReadList = destinationDefinitionsHandler
         .listDestinationDefinitionsForWorkspace(new WorkspaceIdRequestBody().workspaceId(workspaceId));
@@ -209,7 +213,8 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead()
         .destinationDefinitionId(destinationDefinition2.getDestinationDefinitionId())
@@ -222,7 +227,8 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destinationDefinition2.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destinationDefinition2.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final PrivateDestinationDefinitionRead expectedDestinationDefinitionOptInRead1 =
         new PrivateDestinationDefinitionRead().destinationDefinition(expectedDestinationDefinitionRead1).granted(false);
@@ -256,7 +262,8 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody = new DestinationDefinitionIdRequestBody()
         .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId());
@@ -300,7 +307,8 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId = new DestinationDefinitionIdWithWorkspaceId()
         .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId())
@@ -331,7 +339,8 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J
         .icon(destination.getIcon())
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionRead expectedRead = new DestinationDefinitionRead()
         .name(destination.getName())
@@ -343,7 +352,8 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J
         .releaseStage(ReleaseStage.CUSTOM)
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionRead actualRead = destinationDefinitionsHandler.createPrivateDestinationDefinition(create);
 
@@ -373,7 +383,8 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept
         .icon(destination.getIcon())
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final CustomDestinationDefinitionCreate customCreate = new CustomDestinationDefinitionCreate()
         .destinationDefinition(create)
@@ -389,7 +400,8 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept
         .releaseStage(ReleaseStage.CUSTOM)
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final DestinationDefinitionRead actualRead = destinationDefinitionsHandler.createCustomDestinationDefinition(customCreate);
 
@@ -471,7 +483,8 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException,
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
-                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())));
+                .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))
+            .jobSpecific(Collections.emptyList()));
 
     final PrivateDestinationDefinitionRead expectedPrivateDestinationDefinitionRead =
         new PrivateDestinationDefinitionRead().destinationDefinition(expectedDestinationDefinitionRead).granted(true);