From 8078302bc34e242db2b25427a8836a24bf6fb8ff Mon Sep 17 00:00:00 2001 From: Charles Date: Wed, 23 Feb 2022 14:22:00 -0800 Subject: [PATCH] Set resource limits for connector definitions: api layer (#10482) --- airbyte-api/src/main/openapi/config.yaml | 46 ++ .../init/YamlSeedConfigPersistence.java | 9 +- .../seed/destination_definitions.yaml | 16 + .../config/AirbyteConfigValidator.java | 2 + .../ActorDefinitionResourceRequirements.yaml | 7 +- .../resources/types/ResourceRequirements.yaml | 2 +- .../server/converters/ApiPojoConverters.java | 72 +++- .../DestinationDefinitionsHandler.java | 30 +- .../handlers/SourceDefinitionsHandler.java | 21 +- .../DestinationDefinitionsHandlerTest.java | 47 ++- .../server/handlers/SchedulerHandlerTest.java | 18 +- .../SourceDefinitionsHandlerTest.java | 31 +- .../api/generated-api-html/index.html | 399 ++++++++++++++++++ 13 files changed, 646 insertions(+), 54 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index dca784363257..75cdcf81d350 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2124,6 +2124,8 @@ components: format: uri icon: type: string + resourceRequirements: + $ref: "#/components/schemas/ActorDefinitionResourceRequirements" SourceDefinitionUpdate: type: object description: Update the SourceDefinition. Currently, the only allowed attribute to update is the default docker image version. @@ -2135,6 +2137,8 @@ components: $ref: "#/components/schemas/SourceDefinitionId" dockerImageTag: type: string + resourceRequirements: + $ref: "#/components/schemas/ActorDefinitionResourceRequirements" SourceDefinitionRead: type: object required: @@ -2162,6 +2166,8 @@ components: description: The date when this connector was first released, in yyyy-mm-dd format. type: string format: date + resourceRequirements: + $ref: "#/components/schemas/ActorDefinitionResourceRequirements" SourceDefinitionReadList: type: object required: @@ -2393,6 +2399,8 @@ components: format: uri icon: type: string + resourceRequirements: + $ref: "#/components/schemas/ActorDefinitionResourceRequirements" DestinationDefinitionUpdate: type: object required: @@ -2403,6 +2411,8 @@ components: $ref: "#/components/schemas/DestinationDefinitionId" dockerImageTag: type: string + resourceRequirements: + $ref: "#/components/schemas/ActorDefinitionResourceRequirements" DestinationDefinitionRead: type: object required: @@ -2431,6 +2441,8 @@ components: description: The date when this connector was first released, in yyyy-mm-dd format. type: string format: date + resourceRequirements: + $ref: "#/components/schemas/ActorDefinitionResourceRequirements" DestinationDefinitionReadList: type: object required: @@ -3475,6 +3487,40 @@ components: $ref: "#/components/schemas/ConnectionStateObject" ConnectionStateObject: type: object + ActorDefinitionResourceRequirements: + description: actor definition specific resource requirements. if default is set, these are the requirements that should be set for ALL jobs run for this actor definition. it is overriden by the job type specific configurations. if not set, the platform will use defaults. these values will be overriden by configuration at the connection level. + type: object + additionalProperties: false + properties: + default: + "$ref": "#/definitions/ResourceRequirements" + jobSpecific: + type: array + items: + "$ref": "#/definitions/JobTypeResourceLimit" + JobTypeResourceLimit: + description: sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set. + type: object + additionalProperties: false + required: + - jobType + - resourceRequirements + properties: + jobType: + "$ref": "#/definitions/JobType" + resourceRequirements: + "$ref": "#/definitions/ResourceRequirements" + JobType: + description: enum that describes the different types of jobs that the platform runs. + type: string + enum: + - get_spec + - check_connection + - discover_schema + - sync + - reset_connection + - connection_updater + - replicate ResourceRequirements: description: optional resource requirements to run workers (blank for unbounded allocations) type: object diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java index 9c6d32d9e0a4..0603f6f323ac 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/YamlSeedConfigPersistence.java @@ -14,6 +14,7 @@ import io.airbyte.commons.util.MoreIterators; import io.airbyte.commons.yaml.Yamls; import io.airbyte.config.AirbyteConfig; +import io.airbyte.config.AirbyteConfigValidator; import io.airbyte.config.ConfigSchema; import io.airbyte.config.ConfigWithMetadata; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -57,7 +58,9 @@ private YamlSeedConfigPersistence(final Class seedResourceClass) throws IOExc final Map fullSourceDefinitionConfigs = sourceDefinitionConfigs.entrySet().stream() .collect(Collectors.toMap(Entry::getKey, e -> { final JsonNode withTombstone = addMissingTombstoneField(e.getValue()); - return mergeSpecIntoDefinition(withTombstone, sourceSpecConfigs); + final JsonNode output = mergeSpecIntoDefinition(withTombstone, sourceSpecConfigs); + AirbyteConfigValidator.AIRBYTE_CONFIG_VALIDATOR.ensureAsRuntime(ConfigSchema.STANDARD_SOURCE_DEFINITION, output); + return output; })); final Map destinationDefinitionConfigs = getConfigs(seedResourceClass, SeedType.STANDARD_DESTINATION_DEFINITION); @@ -65,7 +68,9 @@ private YamlSeedConfigPersistence(final Class seedResourceClass) throws IOExc final Map fullDestinationDefinitionConfigs = destinationDefinitionConfigs.entrySet().stream() .collect(Collectors.toMap(Entry::getKey, e -> { final JsonNode withTombstone = addMissingTombstoneField(e.getValue()); - return mergeSpecIntoDefinition(withTombstone, destinationSpecConfigs); + final JsonNode output = mergeSpecIntoDefinition(withTombstone, destinationSpecConfigs); + AirbyteConfigValidator.AIRBYTE_CONFIG_VALIDATOR.ensureAsRuntime(ConfigSchema.STANDARD_DESTINATION_DEFINITION, output); + return output; })); this.allSeedConfigs = ImmutableMap.>builder() diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 207c1c88fe9e..f88a92fd6917 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -188,6 +188,22 @@ dockerImageTag: 0.4.15 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg + resourceRequirements: + # this is our first example of setting these requirements. they are guesses, not data driven. + # setting default cpu higher, because we have found that check and discover can be cpu constrained for dbs. + default: + cpu_limit: "1.0" + cpu_request: "1.0" + memory_limit: "300Mi" + memory_request: "300Mi" + jobSpecific: + # sync jobs are generally IO and memory bound and not cpu. + - jobType: sync + resourceRequirements: + cpu_limit: "0.5" + cpu_request: "0.5" + memory_limit: "600Mi" + memory_request: "600Mi" - name: MariaDB ColumnStore destinationDefinitionId: 294a4790-429b-40ae-9516-49826b9702e1 dockerRepository: airbyte/destination-mariadb-columnstore diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteConfigValidator.java b/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteConfigValidator.java index 4acd68d4fd1b..d587d6ddde4a 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteConfigValidator.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteConfigValidator.java @@ -9,6 +9,8 @@ public class AirbyteConfigValidator extends AbstractSchemaValidator { + public static AirbyteConfigValidator AIRBYTE_CONFIG_VALIDATOR = new AirbyteConfigValidator(); + @Override public Path getSchemaPath(final ConfigSchema configType) { return configType.getConfigSchemaFile().toPath(); diff --git a/airbyte-config/models/src/main/resources/types/ActorDefinitionResourceRequirements.yaml b/airbyte-config/models/src/main/resources/types/ActorDefinitionResourceRequirements.yaml index 2d6c4b8c2685..904d6af92683 100644 --- a/airbyte-config/models/src/main/resources/types/ActorDefinitionResourceRequirements.yaml +++ b/airbyte-config/models/src/main/resources/types/ActorDefinitionResourceRequirements.yaml @@ -4,7 +4,8 @@ title: ActorDefinitionResourceRequirements description: actor definition specific resource requirements type: object -additionalProperties: true +# set to false because we need the validations on seeds to be strict. otherwise, we will just add whatever is in the seed file into the db. +additionalProperties: false properties: default: description: if set, these are the requirements that should be set for ALL jobs run for this actor definition. @@ -17,8 +18,10 @@ definitions: JobTypeResourceLimit: description: sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set. type: object + # set to false because we need the validations on seeds to be strict. otherwise, we will just add whatever is in the seed file into the db. + additionalProperties: false required: - - jobtype + - jobType - resourceRequirements properties: jobType: diff --git a/airbyte-config/models/src/main/resources/types/ResourceRequirements.yaml b/airbyte-config/models/src/main/resources/types/ResourceRequirements.yaml index 68fdb68eb108..26f416eba2f8 100644 --- a/airbyte-config/models/src/main/resources/types/ResourceRequirements.yaml +++ b/airbyte-config/models/src/main/resources/types/ResourceRequirements.yaml @@ -4,7 +4,7 @@ title: ResourceRequirements description: generic configuration for pod source requirements type: object -additionalProperties: true +additionalProperties: false properties: # todo (cgardens) - should be camel case for consistency. cpu_request: diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java index d02c3cab743f..3807e0ab30aa 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java @@ -4,33 +4,77 @@ package io.airbyte.server.converters; +import io.airbyte.api.model.ActorDefinitionResourceRequirements; import io.airbyte.api.model.ConnectionRead; import io.airbyte.api.model.ConnectionSchedule; import io.airbyte.api.model.ConnectionStatus; import io.airbyte.api.model.ConnectionUpdate; +import io.airbyte.api.model.JobType; +import io.airbyte.api.model.JobTypeResourceLimit; import io.airbyte.api.model.ResourceRequirements; import io.airbyte.commons.enums.Enums; import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; import io.airbyte.config.Schedule; import io.airbyte.config.StandardSync; import io.airbyte.workers.helper.CatalogConverter; +import java.util.stream.Collectors; public class ApiPojoConverters { - public static io.airbyte.config.ResourceRequirements resourceRequirementsToInternal(final ResourceRequirements resourceRequirements) { + public static io.airbyte.config.ActorDefinitionResourceRequirements actorDefResourceReqsToInternal(final ActorDefinitionResourceRequirements actorDefResourceReqs) { + if (actorDefResourceReqs == null) { + return null; + } + + return new io.airbyte.config.ActorDefinitionResourceRequirements() + .withDefault(actorDefResourceReqs.getDefault() == null ? null : resourceRequirementsToInternal(actorDefResourceReqs.getDefault())) + .withJobSpecific(actorDefResourceReqs.getJobSpecific() == null ? null + : actorDefResourceReqs.getJobSpecific() + .stream() + .map(jobSpecific -> new io.airbyte.config.JobTypeResourceLimit() + .withJobType(toInternalJobType(jobSpecific.getJobType())) + .withResourceRequirements(resourceRequirementsToInternal(jobSpecific.getResourceRequirements()))) + .collect(Collectors.toList())); + } + + public static ActorDefinitionResourceRequirements actorDefResourceReqsToApi(final io.airbyte.config.ActorDefinitionResourceRequirements actorDefResourceReqs) { + if (actorDefResourceReqs == null) { + return null; + } + + return new ActorDefinitionResourceRequirements() + ._default(actorDefResourceReqs.getDefault() == null ? null : resourceRequirementsToApi(actorDefResourceReqs.getDefault())) + .jobSpecific(actorDefResourceReqs.getJobSpecific() == null ? null + : actorDefResourceReqs.getJobSpecific() + .stream() + .map(jobSpecific -> new JobTypeResourceLimit() + .jobType(toApiJobType(jobSpecific.getJobType())) + .resourceRequirements(resourceRequirementsToApi(jobSpecific.getResourceRequirements()))) + .collect(Collectors.toList())); + } + + public static io.airbyte.config.ResourceRequirements resourceRequirementsToInternal(final ResourceRequirements resourceReqs) { + if (resourceReqs == null) { + return null; + } + return new io.airbyte.config.ResourceRequirements() - .withCpuRequest(resourceRequirements.getCpuRequest()) - .withCpuLimit(resourceRequirements.getCpuLimit()) - .withMemoryRequest(resourceRequirements.getMemoryRequest()) - .withMemoryLimit(resourceRequirements.getMemoryLimit()); + .withCpuRequest(resourceReqs.getCpuRequest()) + .withCpuLimit(resourceReqs.getCpuLimit()) + .withMemoryRequest(resourceReqs.getMemoryRequest()) + .withMemoryLimit(resourceReqs.getMemoryLimit()); } - public static ResourceRequirements resourceRequirementsToApi(final io.airbyte.config.ResourceRequirements resourceRequirements) { + public static ResourceRequirements resourceRequirementsToApi(final io.airbyte.config.ResourceRequirements resourceReqs) { + if (resourceReqs == null) { + return null; + } + return new ResourceRequirements() - .cpuRequest(resourceRequirements.getCpuRequest()) - .cpuLimit(resourceRequirements.getCpuLimit()) - .memoryRequest(resourceRequirements.getMemoryRequest()) - .memoryLimit(resourceRequirements.getMemoryLimit()); + .cpuRequest(resourceReqs.getCpuRequest()) + .cpuLimit(resourceReqs.getCpuLimit()) + .memoryRequest(resourceReqs.getMemoryRequest()) + .memoryLimit(resourceReqs.getMemoryLimit()); } public static io.airbyte.config.StandardSync connectionUpdateToInternal(final ConnectionUpdate update) { @@ -90,6 +134,14 @@ public static ConnectionRead internalToConnectionRead(final StandardSync standar return connectionRead; } + public static JobType toApiJobType(final io.airbyte.config.JobTypeResourceLimit.JobType jobType) { + return Enums.convertTo(jobType, JobType.class); + } + + public static io.airbyte.config.JobTypeResourceLimit.JobType toInternalJobType(final JobType jobType) { + return Enums.convertTo(jobType, io.airbyte.config.JobTypeResourceLimit.JobType.class); + } + public static ConnectionSchedule.TimeUnitEnum toApiTimeUnit(final Schedule.TimeUnit apiTimeUnit) { return Enums.convertTo(apiTimeUnit, ConnectionSchedule.TimeUnitEnum.class); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index 1065b31aed01..cb848e5cc432 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -16,12 +16,14 @@ import io.airbyte.api.model.ReleaseStage; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.client.SynchronousResponse; import io.airbyte.scheduler.client.SynchronousSchedulerClient; +import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.server.errors.InternalServerKnownException; import io.airbyte.server.services.AirbyteGithubStore; @@ -77,7 +79,8 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe .documentationUrl(new URI(standardDestinationDefinition.getDocumentationUrl())) .icon(loadIcon(standardDestinationDefinition.getIcon())) .releaseStage(getReleaseStage(standardDestinationDefinition)) - .releaseDate(getReleaseDate(standardDestinationDefinition)); + .releaseDate(getReleaseDate(standardDestinationDefinition)) + .resourceRequirements(ApiPojoConverters.actorDefResourceReqsToApi(standardDestinationDefinition.getResourceRequirements())); } catch (final URISyntaxException | NullPointerException e) { throw new InternalServerKnownException("Unable to process retrieved latest destination definitions list", e); } @@ -127,23 +130,24 @@ public DestinationDefinitionRead getDestinationDefinition(final DestinationDefin configRepository.getStandardDestinationDefinition(destinationDefinitionIdRequestBody.getDestinationDefinitionId())); } - public DestinationDefinitionRead createCustomDestinationDefinition(final DestinationDefinitionCreate destinationDefinitionCreate) + public DestinationDefinitionRead createCustomDestinationDefinition(final DestinationDefinitionCreate destinationDefCreate) throws JsonValidationException, IOException { final ConnectorSpecification spec = getSpecForImage( - destinationDefinitionCreate.getDockerRepository(), - destinationDefinitionCreate.getDockerImageTag()); + destinationDefCreate.getDockerRepository(), + destinationDefCreate.getDockerImageTag()); final UUID id = uuidSupplier.get(); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() .withDestinationDefinitionId(id) - .withDockerRepository(destinationDefinitionCreate.getDockerRepository()) - .withDockerImageTag(destinationDefinitionCreate.getDockerImageTag()) - .withDocumentationUrl(destinationDefinitionCreate.getDocumentationUrl().toString()) - .withName(destinationDefinitionCreate.getName()) - .withIcon(destinationDefinitionCreate.getIcon()) + .withDockerRepository(destinationDefCreate.getDockerRepository()) + .withDockerImageTag(destinationDefCreate.getDockerImageTag()) + .withDocumentationUrl(destinationDefCreate.getDocumentationUrl().toString()) + .withName(destinationDefCreate.getName()) + .withIcon(destinationDefCreate.getIcon()) .withSpec(spec) .withTombstone(false) - .withReleaseStage(StandardDestinationDefinition.ReleaseStage.CUSTOM); + .withReleaseStage(StandardDestinationDefinition.ReleaseStage.CUSTOM) + .withResourceRequirements(ApiPojoConverters.actorDefResourceReqsToInternal(destinationDefCreate.getResourceRequirements())); configRepository.writeStandardDestinationDefinition(destinationDefinition); @@ -162,6 +166,9 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe final ConnectorSpecification spec = specNeedsUpdate ? getSpecForImage(currentDestination.getDockerRepository(), destinationDefinitionUpdate.getDockerImageTag()) : currentDestination.getSpec(); + final ActorDefinitionResourceRequirements updatedResourceReqs = destinationDefinitionUpdate.getResourceRequirements() != null + ? ApiPojoConverters.actorDefResourceReqsToInternal(destinationDefinitionUpdate.getResourceRequirements()) + : currentDestination.getResourceRequirements(); final StandardDestinationDefinition newDestination = new StandardDestinationDefinition() .withDestinationDefinitionId(currentDestination.getDestinationDefinitionId()) @@ -173,7 +180,8 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe .withSpec(spec) .withTombstone(currentDestination.getTombstone()) .withReleaseStage(currentDestination.getReleaseStage()) - .withReleaseDate(currentDestination.getReleaseDate()); + .withReleaseDate(currentDestination.getReleaseDate()) + .withResourceRequirements(updatedResourceReqs); configRepository.writeStandardDestinationDefinition(newDestination); return buildDestinationDefinitionRead(newDestination); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java index 80907169babc..3fbd66b96451 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java @@ -16,12 +16,14 @@ import io.airbyte.api.model.SourceRead; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.client.SynchronousResponse; import io.airbyte.scheduler.client.SynchronousSchedulerClient; +import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.server.errors.InternalServerKnownException; import io.airbyte.server.services.AirbyteGithubStore; @@ -43,15 +45,13 @@ public class SourceDefinitionsHandler { private final SynchronousSchedulerClient schedulerSynchronousClient; private final SourceHandler sourceHandler; - public SourceDefinitionsHandler( - final ConfigRepository configRepository, + public SourceDefinitionsHandler(final ConfigRepository configRepository, final SynchronousSchedulerClient schedulerSynchronousClient, final SourceHandler sourceHandler) { this(configRepository, UUID::randomUUID, schedulerSynchronousClient, AirbyteGithubStore.production(), sourceHandler); } - public SourceDefinitionsHandler( - final ConfigRepository configRepository, + public SourceDefinitionsHandler(final ConfigRepository configRepository, final Supplier uuidSupplier, final SynchronousSchedulerClient schedulerSynchronousClient, final AirbyteGithubStore githubStore, @@ -74,7 +74,9 @@ static SourceDefinitionRead buildSourceDefinitionRead(final StandardSourceDefini .documentationUrl(new URI(standardSourceDefinition.getDocumentationUrl())) .icon(loadIcon(standardSourceDefinition.getIcon())) .releaseStage(getReleaseStage(standardSourceDefinition)) - .releaseDate(getReleaseDate(standardSourceDefinition)); + .releaseDate(getReleaseDate(standardSourceDefinition)) + .resourceRequirements(ApiPojoConverters.actorDefResourceReqsToApi(standardSourceDefinition.getResourceRequirements())); + } catch (final URISyntaxException | NullPointerException e) { throw new InternalServerKnownException("Unable to process retrieved latest source definitions list", e); } @@ -137,7 +139,8 @@ public SourceDefinitionRead createCustomSourceDefinition(final SourceDefinitionC .withIcon(sourceDefinitionCreate.getIcon()) .withSpec(spec) .withTombstone(false) - .withReleaseStage(StandardSourceDefinition.ReleaseStage.CUSTOM); + .withReleaseStage(StandardSourceDefinition.ReleaseStage.CUSTOM) + .withResourceRequirements(ApiPojoConverters.actorDefResourceReqsToInternal(sourceDefinitionCreate.getResourceRequirements())); configRepository.writeStandardSourceDefinition(sourceDefinition); @@ -156,6 +159,9 @@ public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate final ConnectorSpecification spec = specNeedsUpdate ? getSpecForImage(currentSourceDefinition.getDockerRepository(), sourceDefinitionUpdate.getDockerImageTag()) : currentSourceDefinition.getSpec(); + final ActorDefinitionResourceRequirements updatedResourceReqs = sourceDefinitionUpdate.getResourceRequirements() != null + ? ApiPojoConverters.actorDefResourceReqsToInternal(sourceDefinitionUpdate.getResourceRequirements()) + : currentSourceDefinition.getResourceRequirements(); final StandardSourceDefinition newSource = new StandardSourceDefinition() .withSourceDefinitionId(currentSourceDefinition.getSourceDefinitionId()) @@ -167,7 +173,8 @@ public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate .withSpec(spec) .withTombstone(currentSourceDefinition.getTombstone()) .withReleaseStage(currentSourceDefinition.getReleaseStage()) - .withReleaseDate(currentSourceDefinition.getReleaseDate()); + .withReleaseDate(currentSourceDefinition.getReleaseDate()) + .withResourceRequirements(updatedResourceReqs); configRepository.writeStandardSourceDefinition(newSource); return buildSourceDefinitionRead(newSource); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index 402fc952ba57..68aa346f6a21 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -24,7 +24,9 @@ import io.airbyte.api.model.ReleaseStage; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.JobConfig.ConfigType; +import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -68,13 +70,17 @@ void setUp() { githubStore = mock(AirbyteGithubStore.class); destinationHandler = mock(DestinationHandler.class); - destinationDefinitionsHandler = - new DestinationDefinitionsHandler(configRepository, uuidSupplier, schedulerSynchronousClient, githubStore, destinationHandler); + destinationDefinitionsHandler = new DestinationDefinitionsHandler( + configRepository, + uuidSupplier, + schedulerSynchronousClient, + githubStore, + destinationHandler); } private StandardDestinationDefinition generateDestinationDefinition() { - final ConnectorSpecification spec = new ConnectorSpecification().withConnectionSpecification( - Jsons.jsonNode(ImmutableMap.of("foo", "bar"))); + final ConnectorSpecification spec = new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar"))); return new StandardDestinationDefinition() .withDestinationDefinitionId(UUID.randomUUID()) @@ -86,7 +92,8 @@ private StandardDestinationDefinition generateDestinationDefinition() { .withSpec(spec) .withTombstone(false) .withReleaseStage(StandardDestinationDefinition.ReleaseStage.ALPHA) - .withReleaseDate(TODAY_DATE_STRING); + .withReleaseDate(TODAY_DATE_STRING) + .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); } @Test @@ -104,7 +111,10 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())); + .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() .destinationDefinitionId(destination2.getDestinationDefinitionId()) @@ -114,7 +124,10 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt .documentationUrl(new URI(destination2.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destination2.getIcon())) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())); + .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest()))); final DestinationDefinitionReadList actualDestinationDefinitionReadList = destinationDefinitionsHandler.listDestinationDefinitions(); @@ -137,7 +150,10 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())); + .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody = new DestinationDefinitionIdRequestBody() .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()); @@ -164,7 +180,10 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J .dockerRepository(destination.getDockerRepository()) .dockerImageTag(destination.getDockerImageTag()) .documentationUrl(new URI(destination.getDocumentationUrl())) - .icon(destination.getIcon()); + .icon(destination.getIcon()) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))); final DestinationDefinitionRead expectedRead = new DestinationDefinitionRead() .name(destination.getName()) @@ -173,14 +192,18 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J .documentationUrl(new URI(destination.getDocumentationUrl())) .destinationDefinitionId(destination.getDestinationDefinitionId()) .icon(DestinationDefinitionsHandler.loadIcon(destination.getIcon())) - .releaseStage(ReleaseStage.CUSTOM); + .releaseStage(ReleaseStage.CUSTOM) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))); final DestinationDefinitionRead actualRead = destinationDefinitionsHandler.createCustomDestinationDefinition(create); assertEquals(expectedRead, actualRead); verify(schedulerSynchronousClient).createGetSpecJob(imageName); - verify(configRepository).writeStandardDestinationDefinition(destination.withReleaseDate(null).withReleaseStage( - StandardDestinationDefinition.ReleaseStage.CUSTOM)); + verify(configRepository).writeStandardDestinationDefinition(destination + .withReleaseDate(null) + .withReleaseStage(StandardDestinationDefinition.ReleaseStage.CUSTOM)); } @Test diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index 6f3deef17b90..544eb9c903fb 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -514,8 +514,14 @@ void testSyncConnection() throws JsonValidationException, IOException, ConfigNot when(configRepository.getSourceConnection(source.getSourceId())).thenReturn(source); when(configRepository.getDestinationConnection(destination.getDestinationId())).thenReturn(destination); when(configRepository.getStandardSyncOperation(operationId)).thenReturn(getOperation(operationId)); - when(schedulerJobClient.createOrGetActiveSyncJob(source, destination, standardSync, SOURCE_DOCKER_IMAGE, DESTINATION_DOCKER_IMAGE, operations)) - .thenReturn(completedJob); + when(schedulerJobClient.createOrGetActiveSyncJob( + source, + destination, + standardSync, + SOURCE_DOCKER_IMAGE, + DESTINATION_DOCKER_IMAGE, + operations)) + .thenReturn(completedJob); when(completedJob.getScope()).thenReturn("cat:12"); final JobConfig jobConfig = mock(JobConfig.class); when(completedJob.getConfig()).thenReturn(jobConfig); @@ -527,7 +533,13 @@ void testSyncConnection() throws JsonValidationException, IOException, ConfigNot verify(configRepository).getStandardSync(standardSync.getConnectionId()); verify(configRepository).getSourceConnection(standardSync.getSourceId()); verify(configRepository).getDestinationConnection(standardSync.getDestinationId()); - verify(schedulerJobClient).createOrGetActiveSyncJob(source, destination, standardSync, SOURCE_DOCKER_IMAGE, DESTINATION_DOCKER_IMAGE, operations); + verify(schedulerJobClient).createOrGetActiveSyncJob( + source, + destination, + standardSync, + SOURCE_DOCKER_IMAGE, + DESTINATION_DOCKER_IMAGE, + operations); } @Test diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java index 7b6d8e644baa..245ca1067656 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java @@ -25,7 +25,9 @@ import io.airbyte.api.model.SourceReadList; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.JobConfig.ConfigType; +import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -88,7 +90,9 @@ private StandardSourceDefinition generateSourceDefinition() { .withSpec(spec) .withTombstone(false) .withReleaseStage(StandardSourceDefinition.ReleaseStage.ALPHA) - .withReleaseDate(TODAY_DATE_STRING); + .withReleaseDate(TODAY_DATE_STRING) + .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); + } @Test @@ -106,7 +110,10 @@ void testListSourceDefinitions() throws JsonValidationException, IOException, UR .documentationUrl(new URI(sourceDefinition.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) .releaseStage(ReleaseStage.fromValue(sourceDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())); + .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); final SourceDefinitionRead expectedSourceDefinitionRead2 = new SourceDefinitionRead() .sourceDefinitionId(sourceDefinition2.getSourceDefinitionId()) @@ -116,7 +123,10 @@ void testListSourceDefinitions() throws JsonValidationException, IOException, UR .documentationUrl(new URI(sourceDefinition.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) .releaseStage(ReleaseStage.fromValue(sourceDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())); + .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest()))); final SourceDefinitionReadList actualSourceDefinitionReadList = sourceDefinitionsHandler.listSourceDefinitions(); @@ -139,7 +149,10 @@ void testGetSourceDefinition() throws JsonValidationException, ConfigNotFoundExc .documentationUrl(new URI(sourceDefinition.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) .releaseStage(ReleaseStage.fromValue(sourceDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())); + .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody = new SourceDefinitionIdRequestBody().sourceDefinitionId(sourceDefinition.getSourceDefinitionId()); @@ -165,7 +178,10 @@ void testCreateSourceDefinition() throws URISyntaxException, IOException, JsonVa .dockerRepository(sourceDefinition.getDockerRepository()) .dockerImageTag(sourceDefinition.getDockerImageTag()) .documentationUrl(new URI(sourceDefinition.getDocumentationUrl())) - .icon(sourceDefinition.getIcon()); + .icon(sourceDefinition.getIcon()) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); final SourceDefinitionRead expectedRead = new SourceDefinitionRead() .name(sourceDefinition.getName()) @@ -174,7 +190,10 @@ void testCreateSourceDefinition() throws URISyntaxException, IOException, JsonVa .documentationUrl(new URI(sourceDefinition.getDocumentationUrl())) .sourceDefinitionId(sourceDefinition.getSourceDefinitionId()) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) - .releaseStage(ReleaseStage.CUSTOM); + .releaseStage(ReleaseStage.CUSTOM) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); final SourceDefinitionRead actualRead = sourceDefinitionsHandler.createCustomSourceDefinition(create); diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index fc01581c5bd1..9b44f6174600 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -2579,6 +2579,29 @@

Return type

Example data

Content-Type: application/json
{
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
   "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
   "releaseDate" : "2000-01-23",
@@ -2688,6 +2711,29 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
   "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
   "releaseDate" : "2000-01-23",
@@ -2741,6 +2787,29 @@ 
Example data

     
Content-Type: application/json

     
{
   "destinationDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -2749,6 +2818,29 @@ 
Example data

     "icon" : "icon",
     "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -2797,6 +2889,29 @@ 
Example data

     
Content-Type: application/json

     
{
   "destinationDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -2805,6 +2920,29 @@ 
Example data

     "icon" : "icon",
     "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -2864,6 +3002,29 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
   "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
   "releaseDate" : "2000-01-23",
@@ -3227,6 +3388,29 @@ 
Example data

   "job" : {
     "configId" : "configId",
     "sourceDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
       "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
       "releaseDate" : "2000-01-23",
@@ -3238,6 +3422,29 @@ 
Example data

     "airbyteVersion" : "airbyteVersion",
     "id" : 0,
     "destinationDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
       "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
       "releaseDate" : "2000-01-23",
@@ -5609,6 +5816,29 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
   "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
   "releaseDate" : "2000-01-23",
@@ -5718,6 +5948,29 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
   "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
   "releaseDate" : "2000-01-23",
@@ -5771,6 +6024,29 @@ 
Example data

     
Content-Type: application/json

     
{
   "sourceDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -5779,6 +6055,29 @@ 
Example data

     "icon" : "icon",
     "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -5827,6 +6126,29 @@ 
Example data

     
Content-Type: application/json

     
{
   "sourceDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -5835,6 +6157,29 @@ 
Example data

     "icon" : "icon",
     "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
     "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
@@ -5894,6 +6239,29 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
   "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
   "releaseDate" : "2000-01-23",
@@ -7880,6 +8248,7 @@ 
Models

 
   
Table of Contents

   
    
+    
  1. ActorDefinitionResourceRequirements - 
    2. 
     
  2. AdvancedAuth - 
    3. 
     
  3. AirbyteCatalog - 
    4. 
     
  4. AirbyteStream - 
    5. 
@@ -7943,6 +8312,8 @@ 
    Table of Contents
    
     
  5. JobRead - 
    6. 
     
  6. JobReadList - 
    7. 
     
  7. JobStatus - 
    8. 
+    
  8. JobType - 
    9. 
+    
  9. JobTypeResourceLimit - 
    10. 
     
  10. JobWithAttemptsRead - 
    11. 
     
  11. KnownExceptionInfo - 
    12. 
     
  12. LogRead - 
    13. 
@@ -8006,6 +8377,14 @@ 
    Table of Contents
    
     
  13. WorkspaceUpdateName - 
    14. 
   

 
+  

+    
ActorDefinitionResourceRequirements -  Up

+    
actor definition specific resource requirements. if default is set, these are the requirements that should be set for ALL jobs run for this actor definition. it is overriden by the job type specific configurations. if not set, the platform will use defaults. these values will be overriden by configuration at the connection level.

+    

+      
default (optional)
ResourceRequirements  

+
jobSpecific (optional)
array[JobTypeResourceLimit]  

+    
  
+  

   

     
AdvancedAuth -  Up

     

@@ -8371,6 +8750,7 @@ 
DestinationDefinitionCreatedockerImageTag 
String  

 
documentationUrl 
URI  format: uri

 
icon (optional)
String  

+
resourceRequirements (optional)
ActorDefinitionResourceRequirements  

       
   
   

@@ -8392,6 +8772,7 @@ 
DestinationDefinitionRead -
 
icon (optional)
String  

 
releaseStage (optional)
ReleaseStage  

 
releaseDate (optional)
date The date when this connector was first released, in yyyy-mm-dd format. format: date

+
resourceRequirements (optional)
ActorDefinitionResourceRequirements  

     
  
   
   

@@ -8422,6 +8803,7 @@ 
DestinationDefinitionUpdate
       
destinationDefinitionId 
UUID  format: uuid

 
dockerImageTag (optional)
String  

+
resourceRequirements (optional)
ActorDefinitionResourceRequirements  

     
  
   
   

@@ -8607,6 +8989,20 @@ 
JobStatus -  
           
  
   
+  

+    
JobType -  Up

+    
enum that describes the different types of jobs that the platform runs.

+    

+          
  
+  

+  

+    
JobTypeResourceLimit -  Up

+    
sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.

+    

+      
jobType 
JobType  

+
resourceRequirements 
ResourceRequirements  

+    
  
+  

   

     
JobWithAttemptsRead -  Up

     

@@ -8875,6 +9271,7 @@ 
SourceDefinitionCreate - 
 
dockerImageTag 
String  

 
documentationUrl 
URI  format: uri

 
icon (optional)
String  

+
resourceRequirements (optional)
ActorDefinitionResourceRequirements  

     
  
   
   

@@ -8896,6 +9293,7 @@ 
SourceDefinitionRead -  icon (optional)
String  

 
releaseStage (optional)
ReleaseStage  

 
releaseDate (optional)
date The date when this connector was first released, in yyyy-mm-dd format. format: date

+
resourceRequirements (optional)
ActorDefinitionResourceRequirements  

       
   
   

@@ -8923,6 +9321,7 @@ 
SourceDefinitionUpdate - 
     

       
sourceDefinitionId 
UUID  format: uuid

 
dockerImageTag 
String  

+
resourceRequirements (optional)
ActorDefinitionResourceRequirements