diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index c1e41259d894..51112dd3dcaa 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -727,7 +727,6 @@ jobs: ${{ github.workspace }}/airbyte-bootloader/build/reports/jacoco/test/jacocoTestReport.xml, ${{ github.workspace }}/airbyte-commons/build/reports/jacoco/test/jacocoTestReport.xml, ${{ github.workspace }}/airbyte-commons-cli/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/airbyte-commons-docker/build/reports/jacoco/test/jacocoTestReport.xml, ${{ github.workspace }}/airbyte-commons-protocol/build/reports/jacoco/test/jacocoTestReport.xml, ${{ github.workspace }}/airbyte-commons-temporal/build/reports/jacoco/test/jacocoTestReport.xml, ${{ github.workspace }}/airbyte-commons-worker/build/reports/jacoco/test/jacocoTestReport.xml, @@ -1157,7 +1156,7 @@ jobs: helm-acceptance-test: name: "Platform: Acceptance Tests (Helm)" # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. - needs: [start-helm-acceptance-test-runner] # required to start the main job when the runner is ready + needs: [ start-helm-acceptance-test-runner ] # required to start the main job when the runner is ready runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner # this is the label of the runner environment: more-secrets diff --git a/airbyte-commons-docker/build.gradle b/airbyte-commons-docker/build.gradle deleted file mode 100644 index 7fc9a0b2df94..000000000000 --- a/airbyte-commons-docker/build.gradle +++ /dev/null @@ -1,13 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation 'org.apache.commons:commons-compress:1.20' - implementation 'com.github.docker-java:docker-java:3.2.8' - implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' - - testImplementation 'org.apache.commons:commons-lang3:3.11' -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-commons-docker/readme.md b/airbyte-commons-docker/readme.md deleted file mode 100644 index 46ed14b70f14..000000000000 --- a/airbyte-commons-docker/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# airbyte-commons-docker - -This module contains common helpers for interacting with Docker and Docker images from Java. diff --git a/airbyte-commons-docker/src/main/java/io/airbyte/commons/docker/DockerUtils.java b/airbyte-commons-docker/src/main/java/io/airbyte/commons/docker/DockerUtils.java deleted file mode 100644 index 254e81f483d8..000000000000 --- a/airbyte-commons-docker/src/main/java/io/airbyte/commons/docker/DockerUtils.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.BuildImageResultCallback; -import com.github.dockerjava.core.DefaultDockerClientConfig; -import com.github.dockerjava.core.DockerClientConfig; -import com.github.dockerjava.core.DockerClientImpl; -import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; -import com.github.dockerjava.transport.DockerHttpClient; -import java.io.File; -import java.util.Set; - -public class DockerUtils { - - private static final DockerClientConfig CONFIG = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); - private static final DockerHttpClient HTTP_CLIENT = new ApacheDockerHttpClient.Builder() - .dockerHost(CONFIG.getDockerHost()) - .sslConfig(CONFIG.getSSLConfig()) - .maxConnections(100) - .build(); - private static final DockerClient DOCKER_CLIENT = DockerClientImpl.getInstance(CONFIG, HTTP_CLIENT); - - public static String getTaggedImageName(final String dockerRepository, final String tag) { - return String.join(":", dockerRepository, tag); - } - - public static String buildImage(final String dockerFilePath, final String tag) { - return DOCKER_CLIENT.buildImageCmd() - .withDockerfile(new File(dockerFilePath)) - .withTags(Set.of(tag)) - .exec(new BuildImageResultCallback()) - .awaitImageId(); - } - -} diff --git a/airbyte-commons-docker/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java b/airbyte-commons-docker/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java deleted file mode 100644 index ebb013d85fcd..000000000000 --- a/airbyte-commons-docker/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.docker; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -class DockerUtilsTest { - - @Test - void testGetTaggedImageName() { - final String repository = "airbyte/repo"; - final String tag = "12.3"; - assertEquals("airbyte/repo:12.3", DockerUtils.getTaggedImageName(repository, tag)); - } - -} diff --git a/airbyte-commons-server/build.gradle b/airbyte-commons-server/build.gradle index f0ea82dcb4e5..9a3557867d3c 100644 --- a/airbyte-commons-server/build.gradle +++ b/airbyte-commons-server/build.gradle @@ -29,7 +29,6 @@ dependencies { implementation project(':airbyte-analytics') implementation project(':airbyte-api') - implementation project(':airbyte-commons-docker') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:init') @@ -66,4 +65,4 @@ dependencies { testImplementation 'org.mockito:mockito-inline:4.7.0' } -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) \ No newline at end of file +Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java index 813cda084b55..6c1032c4d311 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java @@ -17,7 +17,6 @@ import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList; import io.airbyte.api.model.generated.ReleaseStage; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.server.ServerConstants; import io.airbyte.commons.server.converters.ApiPojoConverters; @@ -32,7 +31,11 @@ import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.commons.version.AirbyteProtocolVersionRange; import io.airbyte.commons.version.Version; -import io.airbyte.config.*; +import io.airbyte.config.ActorDefinitionResourceRequirements; +import io.airbyte.config.ActorType; +import io.airbyte.config.Configs; +import io.airbyte.config.EnvConfigs; +import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; @@ -295,7 +298,7 @@ public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody private ConnectorSpecification getSpecForImage(final String dockerRepository, final String imageTag, final boolean isCustomConnector) throws IOException { - final String imageName = DockerUtils.getTaggedImageName(dockerRepository, imageTag); + final String imageName = dockerRepository + ":" + imageTag; final SynchronousResponse getSpecResponse = schedulerSynchronousClient.createGetSpecJob(imageName, isCustomConnector); return SpecFetcher.getSpecFromJob(getSpecResponse); } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index e0bd34451a7c..21591eb9e2c6 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -41,7 +41,6 @@ import io.airbyte.api.model.generated.StreamTransform; import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum; import io.airbyte.api.model.generated.SynchronousJobRead; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.json.Jsons; @@ -153,7 +152,7 @@ public CheckConnectionRead checkSourceConnectionFromSourceId(final SourceIdReque throws ConfigNotFoundException, IOException, JsonValidationException { final SourceConnection source = configRepository.getSourceConnection(sourceIdRequestBody.getSourceId()); final StandardSourceDefinition sourceDef = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId()); - final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag()); + final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag(); final boolean isCustomConnector = sourceDef.getCustom(); final Version protocolVersion = new Version(sourceDef.getProtocolVersion()); @@ -177,7 +176,7 @@ public CheckConnectionRead checkSourceConnectionFromSourceCreate(final SourceCor final Version protocolVersion = new Version(sourceDef.getProtocolVersion()); - final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag()); + final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag(); final boolean isCustomConnector = sourceDef.getCustom(); return reportConnectionStatus(synchronousSchedulerClient.createSourceCheckConnectionJob(source, imageName, protocolVersion, isCustomConnector)); } @@ -202,7 +201,7 @@ public CheckConnectionRead checkDestinationConnectionFromDestinationId(final Des throws ConfigNotFoundException, IOException, JsonValidationException { final DestinationConnection destination = configRepository.getDestinationConnection(destinationIdRequestBody.getDestinationId()); final StandardDestinationDefinition destinationDef = configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId()); - final String imageName = DockerUtils.getTaggedImageName(destinationDef.getDockerRepository(), destinationDef.getDockerImageTag()); + final String imageName = destinationDef.getDockerRepository() + ":" + destinationDef.getDockerImageTag(); final boolean isCustomConnector = destinationDef.getCustom(); final Version protocolVersion = new Version(destinationDef.getProtocolVersion()); return reportConnectionStatus( @@ -224,7 +223,7 @@ public CheckConnectionRead checkDestinationConnectionFromDestinationCreate(final .withDestinationDefinitionId(destinationConfig.getDestinationDefinitionId()) .withConfiguration(partialConfig) .withWorkspaceId(destinationConfig.getWorkspaceId()); - final String imageName = DockerUtils.getTaggedImageName(destDef.getDockerRepository(), destDef.getDockerImageTag()); + final String imageName = destDef.getDockerRepository() + ":" + destDef.getDockerImageTag(); final Version protocolVersion = new Version(destDef.getProtocolVersion()); return reportConnectionStatus( synchronousSchedulerClient.createDestinationCheckConnectionJob(destination, imageName, protocolVersion, isCustomConnector)); @@ -250,7 +249,7 @@ public SourceDiscoverSchemaRead discoverSchemaForSourceFromSourceId(final Source throws ConfigNotFoundException, IOException, JsonValidationException { final SourceConnection source = configRepository.getSourceConnection(discoverSchemaRequestBody.getSourceId()); final StandardSourceDefinition sourceDef = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId()); - final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag()); + final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag(); final boolean isCustomConnector = sourceDef.getCustom(); final String configHash = HASH_FUNCTION.hashBytes(Jsons.serialize(source.getConfiguration()).getBytes( @@ -298,7 +297,7 @@ public SourceDiscoverSchemaRead discoverSchemaForSourceFromSourceCreate(final So sourceCreate.getConnectionConfiguration(), sourceDef.getSpec()); - final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag()); + final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag(); final boolean isCustomConnector = sourceDef.getCustom(); // todo (cgardens) - narrow the struct passed to the client. we are not setting fields that are // technically declared as required. diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java index 9ee876cba701..8908141fa0c1 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java @@ -18,7 +18,6 @@ import io.airbyte.api.model.generated.SourceDefinitionUpdate; import io.airbyte.api.model.generated.SourceRead; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.server.ServerConstants; import io.airbyte.commons.server.converters.ApiPojoConverters; @@ -33,7 +32,11 @@ import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.commons.version.AirbyteProtocolVersionRange; import io.airbyte.commons.version.Version; -import io.airbyte.config.*; +import io.airbyte.config.ActorDefinitionResourceRequirements; +import io.airbyte.config.ActorType; +import io.airbyte.config.Configs; +import io.airbyte.config.EnvConfigs; +import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; @@ -296,7 +299,7 @@ public void deleteSourceDefinition(final SourceDefinitionIdRequestBody sourceDef private ConnectorSpecification getSpecForImage(final String dockerRepository, final String imageTag, final boolean isCustomConnector) throws IOException { - final String imageName = DockerUtils.getTaggedImageName(dockerRepository, imageTag); + final String imageName = dockerRepository + ":" + imageTag; final SynchronousResponse getSpecResponse = schedulerSynchronousClient.createGetSpecJob(imageName, isCustomConnector); return SpecFetcher.getSpecFromJob(getSpecResponse); } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java index 8848e7cf4f22..b621286d017b 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java @@ -29,7 +29,6 @@ import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList; import io.airbyte.api.model.generated.ReleaseStage; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.errors.IdNotFoundKnownException; import io.airbyte.commons.server.errors.UnsupportedProtocolVersionException; @@ -382,7 +381,7 @@ void testCreateDestinationDefinitionShouldCheckProtocolVersion() throws URISynta final String invalidProtocolVersion = "121.5.6"; final StandardDestinationDefinition destination = generateDestinationDefinition(); destination.getSpec().setProtocolVersion(invalidProtocolVersion); - final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final String imageName = destination.getDockerRepository() + ":" + destination.getDockerImageTag(); when(uuidSupplier.get()).thenReturn(destination.getDestinationDefinitionId()); when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>( @@ -417,7 +416,7 @@ void testCreateDestinationDefinitionShouldCheckProtocolVersion() throws URISynta @DisplayName("createCustomDestinationDefinition should correctly create a destinationDefinition") void testCreateCustomDestinationDefinition() throws URISyntaxException, IOException, JsonValidationException { final StandardDestinationDefinition destination = generateDestinationDefinition(); - final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final String imageName = destination.getDockerRepository() + ":" + destination.getDockerImageTag(); when(uuidSupplier.get()).thenReturn(destination.getDestinationDefinitionId()); when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>( @@ -474,7 +473,7 @@ void testCreateCustomDestinationDefinitionWithInvalidProtocol() throws URISyntax final String invalidProtocol = "122.1.22"; final StandardDestinationDefinition destination = generateDestinationDefinition(); destination.getSpec().setProtocolVersion(invalidProtocol); - final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final String imageName = destination.getDockerRepository() + ":" + destination.getDockerImageTag(); when(uuidSupplier.get()).thenReturn(destination.getDestinationDefinitionId()); when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>( @@ -521,7 +520,7 @@ void testUpdateDestination() throws ConfigNotFoundException, IOException, JsonVa assertNotEquals(newDockerImageTag, currentTag); assertNotEquals(newProtocolVersion, currentDestination.getProtocolVersion()); - final String newImageName = DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), newDockerImageTag); + final String newImageName = destinationDefinition.getDockerRepository() + ":" + newDockerImageTag; final ConnectorSpecification newSpec = new ConnectorSpecification() .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2"))) .withProtocolVersion(newProtocolVersion); @@ -557,7 +556,7 @@ void testOutOfProtocolRangeUpdateDestination() throws ConfigNotFoundException, I assertNotEquals(newDockerImageTag, currentTag); assertNotEquals(newProtocolVersion, currentDestination.getProtocolVersion()); - final String newImageName = DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), newDockerImageTag); + final String newImageName = destinationDefinition.getDockerRepository() + ":" + newDockerImageTag; final ConnectorSpecification newSpec = new ConnectorSpecification() .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2"))) .withProtocolVersion(newProtocolVersion); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java index 5b4e2773112a..64a087fd4c70 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java @@ -47,7 +47,6 @@ import io.airbyte.api.model.generated.SourceUpdate; import io.airbyte.api.model.generated.StreamTransform; import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; @@ -103,12 +102,12 @@ class SchedulerHandlerTest { private static final String SOURCE_DOCKER_REPO = "srcimage"; private static final String SOURCE_DOCKER_TAG = "tag"; - private static final String SOURCE_DOCKER_IMAGE = DockerUtils.getTaggedImageName(SOURCE_DOCKER_REPO, SOURCE_DOCKER_TAG); + private static final String SOURCE_DOCKER_IMAGE = SOURCE_DOCKER_REPO + ":" + SOURCE_DOCKER_TAG; private static final String SOURCE_PROTOCOL_VERSION = "0.4.5"; private static final String DESTINATION_DOCKER_REPO = "dstimage"; private static final String DESTINATION_DOCKER_TAG = "tag"; - private static final String DESTINATION_DOCKER_IMAGE = DockerUtils.getTaggedImageName(DESTINATION_DOCKER_REPO, DESTINATION_DOCKER_TAG); + private static final String DESTINATION_DOCKER_IMAGE = DESTINATION_DOCKER_REPO + ":" + DESTINATION_DOCKER_TAG; private static final String DESTINATION_PROTOCOL_VERSION = "0.7.9"; private static final String NAME = "name"; private static final String DOGS = "dogs"; diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java index 0fc880fbfc33..7a67dc01fe0b 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java @@ -31,7 +31,6 @@ import io.airbyte.api.model.generated.SourceRead; import io.airbyte.api.model.generated.SourceReadList; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.errors.IdNotFoundKnownException; import io.airbyte.commons.server.errors.UnsupportedProtocolVersionException; @@ -333,7 +332,7 @@ void testCreateSourceDefinitionWithInvalidProtocol() throws URISyntaxException, final String invalidProtocol = "131.1.2"; final StandardSourceDefinition sourceDefinition = generateSourceDefinition(); sourceDefinition.getSpec().setProtocolVersion(invalidProtocol); - final String imageName = DockerUtils.getTaggedImageName(sourceDefinition.getDockerRepository(), sourceDefinition.getDockerImageTag()); + final String imageName = sourceDefinition.getDockerRepository() + ":" + sourceDefinition.getDockerImageTag(); when(uuidSupplier.get()).thenReturn(sourceDefinition.getSourceDefinitionId()); when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>( @@ -368,7 +367,7 @@ void testCreateSourceDefinitionWithInvalidProtocol() throws URISyntaxException, @DisplayName("createCustomSourceDefinition should correctly create a sourceDefinition") void testCreateCustomSourceDefinition() throws URISyntaxException, IOException, JsonValidationException { final StandardSourceDefinition sourceDefinition = generateSourceDefinition(); - final String imageName = DockerUtils.getTaggedImageName(sourceDefinition.getDockerRepository(), sourceDefinition.getDockerImageTag()); + final String imageName = sourceDefinition.getDockerRepository() + ":" + sourceDefinition.getDockerImageTag(); when(uuidSupplier.get()).thenReturn(sourceDefinition.getSourceDefinitionId()); when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>( @@ -423,7 +422,7 @@ void testCreateCustomSourceDefinitionWithInvalidProtocol() throws URISyntaxExcep final String invalidVersion = "130.0.0"; final StandardSourceDefinition sourceDefinition = generateSourceDefinition(); sourceDefinition.getSpec().setProtocolVersion(invalidVersion); - final String imageName = DockerUtils.getTaggedImageName(sourceDefinition.getDockerRepository(), sourceDefinition.getDockerImageTag()); + final String imageName = sourceDefinition.getDockerRepository() + ":" + sourceDefinition.getDockerImageTag(); when(uuidSupplier.get()).thenReturn(sourceDefinition.getSourceDefinitionId()); when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>( @@ -468,7 +467,7 @@ void testUpdateSourceDefinition() throws ConfigNotFoundException, IOException, J final String currentTag = sourceDefinition.getDockerImageTag(); assertNotEquals(newDockerImageTag, currentTag); - final String newImageName = DockerUtils.getTaggedImageName(this.sourceDefinition.getDockerRepository(), newDockerImageTag); + final String newImageName = this.sourceDefinition.getDockerRepository() + ":" + newDockerImageTag; final ConnectorSpecification newSpec = new ConnectorSpecification() .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2"))) .withProtocolVersion(newProtocolVersion); @@ -501,7 +500,7 @@ void testUpdateSourceDefinitionWithInvalidProtocol() throws ConfigNotFoundExcept final String currentTag = sourceDefinition.getDockerImageTag(); assertNotEquals(newDockerImageTag, currentTag); - final String newImageName = DockerUtils.getTaggedImageName(this.sourceDefinition.getDockerRepository(), newDockerImageTag); + final String newImageName = this.sourceDefinition.getDockerRepository() + ":" + newDockerImageTag; final ConnectorSpecification newSpec = new ConnectorSpecification() .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2"))) .withProtocolVersion(newProtocolVersion); diff --git a/airbyte-commons-worker/build.gradle b/airbyte-commons-worker/build.gradle index a79b86345c4d..7b64271baac2 100644 --- a/airbyte-commons-worker/build.gradle +++ b/airbyte-commons-worker/build.gradle @@ -43,8 +43,8 @@ dependencies { testImplementation libs.platform.testcontainers.postgresql testImplementation libs.jmh.core testImplementation libs.jmh.annotations - - testImplementation project(':airbyte-commons-docker') + testImplementation 'com.github.docker-java:docker-java:3.2.8' + testImplementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' } Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java index 7fb14d19948b..4de388d00c6a 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.LoggingHelper.Color; @@ -89,7 +88,7 @@ void setup() throws IOException, WorkerException { WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog)); when(processFactory.create(NORMALIZE_STEP, JOB_ID, JOB_ATTEMPT, jobRoot, - DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), false, false, files, null, + getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), false, false, files, null, workerConfigs.getResourceRequirements(), null, Map.of(JOB_TYPE_KEY, SYNC_JOB, SYNC_STEP_KEY, NORMALIZE_STEP), @@ -117,7 +116,7 @@ public void tearDown() throws IOException { @Test void test() throws Exception { final NormalizationRunner runner = - new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); + new DefaultNormalizationRunner(processFactory, getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); when(process.exitValue()).thenReturn(0); @@ -128,7 +127,7 @@ void test() throws Exception { void testLog() throws Exception { final NormalizationRunner runner = - new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); + new DefaultNormalizationRunner(processFactory, getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); when(process.exitValue()).thenReturn(0); @@ -150,7 +149,7 @@ void testClose() throws Exception { when(process.isAlive()).thenReturn(true).thenReturn(false); final NormalizationRunner runner = - new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); + new DefaultNormalizationRunner(processFactory, getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()); runner.close(); @@ -162,7 +161,7 @@ void testFailure() throws Exception { when(process.exitValue()).thenReturn(1); final NormalizationRunner runner = - new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); + new DefaultNormalizationRunner(processFactory, getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); verify(process).waitFor(); @@ -182,8 +181,8 @@ void testFailureWithTraceMessage() throws Exception { """.replace("\n", ""); when(process.getInputStream()).thenReturn(new ByteArrayInputStream(errorTraceString.getBytes(StandardCharsets.UTF_8))); - final NormalizationRunner runner = - new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), + INTEGRATION_TYPE); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); assertEquals(1, runner.getTraceMessages().count()); @@ -209,7 +208,7 @@ void testFailureWithDbtError() throws Exception { when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8))); final NormalizationRunner runner = - new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); + new DefaultNormalizationRunner(processFactory, getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); assertEquals(1, runner.getTraceMessages().count()); @@ -230,7 +229,7 @@ void testFailureWithDbtErrorJsonFormat() throws Exception { when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8))); final NormalizationRunner runner = - new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); + new DefaultNormalizationRunner(processFactory, getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); assertEquals(1, runner.getTraceMessages().count()); @@ -240,4 +239,8 @@ void testFailureWithDbtErrorJsonFormat() throws Exception { assertThrows(WorkerException.class, runner::close); } + static String getTaggedImageName(final String repository, final String tag) { + return repository + ":" + tag; + } + } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/KubePodProcessTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/KubePodProcessTest.java index 5d324690da0c..7bd9243943cd 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/KubePodProcessTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/KubePodProcessTest.java @@ -7,19 +7,27 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import io.airbyte.commons.docker.DockerUtils; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.BuildImageResultCallback; +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; +import com.github.dockerjava.transport.DockerHttpClient; import io.airbyte.commons.string.Strings; import io.fabric8.kubernetes.api.model.ContainerBuilder; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.KubernetesClient; +import java.io.File; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.github.dockerjava.core.DefaultDockerClientConfig; +import org.testcontainers.shaded.com.github.dockerjava.core.DockerClientConfig; +import org.testcontainers.shaded.com.github.dockerjava.core.DockerClientImpl; import org.testcontainers.shaded.com.google.common.io.Resources; // Disabled until we start minikube on the node. @@ -34,6 +42,26 @@ class KubePodProcessTest { private static final String TEST_IMAGE_NO_VAR_PATH = "Dockerfile.no_var"; private static final String TEST_IMAGE_NO_VAR_NAME = "worker-test:no-var"; + private class DockerUtils { + + private static final DockerClientConfig CONFIG = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); + private static final DockerHttpClient HTTP_CLIENT = new ApacheDockerHttpClient.Builder() + .dockerHost(CONFIG.getDockerHost()) + .sslConfig(CONFIG.getSSLConfig()) + .maxConnections(100) + .build(); + private static final DockerClient DOCKER_CLIENT = DockerClientImpl.getInstance(CONFIG, HTTP_CLIENT); + + public static String buildImage(final String dockerFilePath, final String tag) { + return DOCKER_CLIENT.buildImageCmd() + .withDockerfile(new File(dockerFilePath)) + .withTags(Set.of(tag)) + .exec(new BuildImageResultCallback()) + .awaitImageId(); + } + + } + @BeforeAll static void setup() { final var varDockerfile = Resources.getResource(TEST_IMAGE_WITH_VAR_PATH); diff --git a/airbyte-commons/readme.md b/airbyte-commons/readme.md index acbd8542866a..1f953216d630 100644 --- a/airbyte-commons/readme.md +++ b/airbyte-commons/readme.md @@ -2,6 +2,10 @@ Common java helpers. -This submodule is inherited by all other java modules in the monorepo! It is therefore important that we do not add dependencies to it, as those dependencies will also be added to every java module. The only dependencies that this module uses are the ones declared in the `build.gradle` at the root of the Airbyte monorepo. In other words it only uses dependencies that are already shared across all modules. The `dependencies` section of the `build.gradle` of `airbyte-commons` should always be empty. +This submodule is inherited by all other java modules in the monorepo! It is therefore important that we do not add dependencies to it, as those +dependencies will also be added to every java module. The only dependencies that this module uses are the ones declared in the `build.gradle` at the +root of the Airbyte monorepo. In other words it only uses dependencies that are already shared across all modules. The `dependencies` section of +the `build.gradle` of `airbyte-commons` should always be empty. -For other common java code that needs to be shared across modules that requires additional dependencies, we follow this convention: `airbyte-commons-`. See for example `airbyte-commons-cli` and `airbyte-commons-docker`. +For other common java code that needs to be shared across modules that requires additional dependencies, we follow this +convention: `airbyte-commons-`. See for example `airbyte-commons-cli`. diff --git a/airbyte-config/config-persistence/build.gradle b/airbyte-config/config-persistence/build.gradle index a4013408ed42..48c2772aa34e 100644 --- a/airbyte-config/config-persistence/build.gradle +++ b/airbyte-config/config-persistence/build.gradle @@ -9,7 +9,6 @@ configurations.all { dependencies { implementation project(':airbyte-commons') - implementation project(':airbyte-commons-docker') implementation project(':airbyte-commons-protocol') implementation project(':airbyte-config:config-models') implementation project(':airbyte-db:db-lib') diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 02db19c1b14d..5de81b22861d 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -11,7 +11,6 @@ dependencies { implementation project(':airbyte-config:config-persistence') implementation project(':airbyte-persistence:job-persistence') implementation project(':airbyte-protocol:protocol-models') - implementation project(':airbyte-commons-docker') implementation project(':airbyte-json-validation') implementation libs.lombok implementation libs.micronaut.cache.caffeine diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java index 96b6efc6f2ae..30e27c2eb425 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/LocalDefinitionsProvider.java @@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.io.Resources; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.commons.version.AirbyteProtocolVersion; @@ -131,9 +130,7 @@ private static Map getJsonElements(final Class seedDefiniti * @return JSON of connector definition including the connector spec */ private static JsonNode mergeSpecIntoDefinition(final JsonNode definitionJson, final Map specConfigs) { - final String dockerImage = DockerUtils.getTaggedImageName( - definitionJson.get("dockerRepository").asText(), - definitionJson.get("dockerImageTag").asText()); + final String dockerImage = definitionJson.get("dockerRepository").asText() + ":" + definitionJson.get("dockerImageTag").asText(); final JsonNode specConfigJson = specConfigs.get(dockerImage); if (specConfigJson == null || specConfigJson.get(SPEC) == null) { throw new UnsupportedOperationException(String.format("There is no seed spec for docker image %s", dockerImage)); diff --git a/airbyte-config/specs/build.gradle b/airbyte-config/specs/build.gradle index e278115b6b78..123fee561a37 100644 --- a/airbyte-config/specs/build.gradle +++ b/airbyte-config/specs/build.gradle @@ -7,7 +7,6 @@ dependencies { implementation project(':airbyte-commons') implementation project(':airbyte-commons-cli') - implementation project(':airbyte-commons-docker') implementation project(':airbyte-config:config-models') implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-json-validation') diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/CombinedConnectorCatalogGenerator.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/CombinedConnectorCatalogGenerator.java index a208020bba01..01e5f44818ca 100644 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/CombinedConnectorCatalogGenerator.java +++ b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/CombinedConnectorCatalogGenerator.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.cli.Clis; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; @@ -85,9 +84,7 @@ void mergeSpecsIntoDefinitions(final List definitions, final List json.get("spec"))); for (final JsonNode definition : definitions) { - final String dockerImage = DockerUtils.getTaggedImageName( - definition.get("dockerRepository").asText(), - definition.get("dockerImageTag").asText()); + final String dockerImage = definition.get("dockerRepository").asText() + ":" + definition.get("dockerImageTag").asText(); final JsonNode specConfigJson = specsByImage.get(dockerImage); if (specConfigJson == null) { diff --git a/airbyte-container-orchestrator/build.gradle b/airbyte-container-orchestrator/build.gradle index 1845fa3c7fe6..473aae7e5eed 100644 --- a/airbyte-container-orchestrator/build.gradle +++ b/airbyte-container-orchestrator/build.gradle @@ -48,8 +48,6 @@ dependencies { testImplementation libs.postgresql testImplementation libs.platform.testcontainers testImplementation libs.platform.testcontainers.postgresql - - testImplementation project(':airbyte-commons-docker') } application { diff --git a/airbyte-persistence/job-persistence/build.gradle b/airbyte-persistence/job-persistence/build.gradle index e4c1f4e89247..3ae3648b5488 100644 --- a/airbyte-persistence/job-persistence/build.gradle +++ b/airbyte-persistence/job-persistence/build.gradle @@ -4,7 +4,6 @@ plugins { dependencies { implementation project(':airbyte-commons') - implementation project(':airbyte-commons-docker') implementation project(':airbyte-commons-protocol') implementation project(':airbyte-oauth') implementation project(':airbyte-config:config-models') diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java index 71544885062f..d44fd76ce341 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java @@ -6,7 +6,6 @@ import com.google.common.collect.ImmutableSet; import edu.umd.cs.findbugs.annotations.Nullable; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.map.MoreMaps; import io.airbyte.config.AttemptFailureSummary; @@ -117,8 +116,9 @@ public void reportSyncJobFailure(final UUID connectionId, final AttemptFailureSu getNormalizationMetadata(destinationDefinition.getNormalizationConfig().getNormalizationRepository()), prefixConnectorMetadataKeys(getSourceMetadata(sourceDefinition), "source"), getDestinationMetadata(destinationDefinition)); - final String dockerImage = DockerUtils.getTaggedImageName(destinationDefinition.getNormalizationConfig().getNormalizationRepository(), - destinationDefinition.getNormalizationConfig().getNormalizationTag()); + final String dockerImage = + destinationDefinition.getNormalizationConfig().getNormalizationRepository() + ":" + + destinationDefinition.getNormalizationConfig().getNormalizationTag(); reportJobFailureReason(workspace, failureReason, dockerImage, metadata); } @@ -225,7 +225,7 @@ private Map getSourceMetadata(final StandardSourceDefinition sou Map.entry(CONNECTOR_RELEASE_STAGE_META_KEY, sourceDefinition.getReleaseStage().value())); } - private Map getNormalizationMetadata(String normalizationImage) { + private Map getNormalizationMetadata(final String normalizationImage) { return Map.ofEntries( Map.entry(NORMALIZATION_REPOSITORY_META_KEY, normalizationImage)); } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java index e7a69eb64533..48539bdd9438 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.version.Version; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; @@ -67,9 +66,8 @@ public Long create(final UUID connectionId) { final StandardDestinationDefinition destinationDefinition = configRepository .getStandardDestinationDefinition(destinationConnection.getDestinationDefinitionId()); - final String sourceImageName = DockerUtils.getTaggedImageName(sourceDefinition.getDockerRepository(), sourceDefinition.getDockerImageTag()); - final String destinationImageName = - DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), destinationDefinition.getDockerImageTag()); + final String sourceImageName = sourceDefinition.getDockerRepository() + ":" + sourceDefinition.getDockerImageTag(); + final String destinationImageName = destinationDefinition.getDockerRepository() + ":" + destinationDefinition.getDockerImageTag(); final List standardSyncOperations = Lists.newArrayList(); for (final var operationId : standardSync.getOperationIds()) { diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java index 5c4e439e5542..ed8603ee40fa 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java @@ -12,7 +12,6 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.version.Version; import io.airbyte.config.DestinationConnection; @@ -67,12 +66,12 @@ void createSyncJobFromConnectionId() throws JsonValidationException, ConfigNotFo final String srcDockerRepo = "srcrepo"; final String srcDockerTag = "tag"; - final String srcDockerImage = DockerUtils.getTaggedImageName(srcDockerRepo, srcDockerTag); + final String srcDockerImage = srcDockerRepo + ":" + srcDockerTag; final Version srcProtocolVersion = new Version("0.3.1"); final String dstDockerRepo = "dstrepo"; final String dstDockerTag = "tag"; - final String dstDockerImage = DockerUtils.getTaggedImageName(dstDockerRepo, dstDockerTag); + final String dstDockerImage = dstDockerRepo + ":" + dstDockerTag; final Version dstProtocolVersion = new Version("0.3.2"); final StandardSourceDefinition standardSourceDefinition = new StandardSourceDefinition().withSourceDefinitionId(sourceDefinitionId).withDockerRepository(srcDockerRepo) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index c75ccaf1add0..3d4ea0c87367 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -33,7 +33,6 @@ dependencies { implementation project(':airbyte-analytics') implementation project(':airbyte-api') - implementation project(':airbyte-commons-docker') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-commons-server') implementation project(':airbyte-commons-worker') diff --git a/airbyte-tests/build.gradle b/airbyte-tests/build.gradle index 038267ab977f..23084735b098 100644 --- a/airbyte-tests/build.gradle +++ b/airbyte-tests/build.gradle @@ -57,6 +57,8 @@ dependencies { acceptanceTestsImplementation 'org.apache.commons:commons-csv:1.4' acceptanceTestsImplementation libs.platform.testcontainers.postgresql acceptanceTestsImplementation libs.postgresql + acceptanceTestsImplementation 'org.bouncycastle:bcprov-jdk15on:1.66' + acceptanceTestsImplementation 'org.bouncycastle:bcpkix-jdk15on:1.66' automaticMigrationAcceptanceTestImplementation project(':airbyte-api') automaticMigrationAcceptanceTestImplementation project(':airbyte-commons') diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 2234d746c251..ccffeddba358 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -55,7 +55,6 @@ dependencies { implementation project(':airbyte-analytics') implementation project(':airbyte-api') - implementation project(':airbyte-commons-docker') implementation project(':airbyte-commons-protocol') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-commons-worker') @@ -92,9 +91,9 @@ dependencies { testImplementation libs.postgresql testImplementation libs.platform.testcontainers testImplementation libs.platform.testcontainers.postgresql - - testImplementation project(':airbyte-commons-docker') testImplementation project(':airbyte-test-utils') + testImplementation 'org.bouncycastle:bcprov-jdk15on:1.66' + testImplementation 'org.bouncycastle:bcpkix-jdk15on:1.66' integrationTestJavaImplementation project(':airbyte-workers') integrationTestJavaImplementation libs.bundles.micronaut.test diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index 403c70fe33d6..2f03d0735783 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -9,7 +9,6 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; import datadog.trace.api.Trace; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalWorkflowUtils; import io.airbyte.commons.temporal.config.WorkerMode; @@ -107,8 +106,8 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { final StandardDestinationDefinition destinationDefinition = configRepository.getDestinationDefinitionFromDestination(standardSync.getDestinationId()); final String destinationNormalizationDockerImage = destinationDefinition.getNormalizationConfig() != null - ? DockerUtils.getTaggedImageName(destinationDefinition.getNormalizationConfig().getNormalizationRepository(), - destinationDefinition.getNormalizationConfig().getNormalizationTag()) + ? destinationDefinition.getNormalizationConfig().getNormalizationRepository() + ":" + + destinationDefinition.getNormalizationConfig().getNormalizationTag() : null; final String normalizationIntegrationType = destinationDefinition.getNormalizationConfig() != null ? destinationDefinition.getNormalizationConfig().getNormalizationIntegrationType() diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java index 9915ccbca2bc..33601d90ce54 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java @@ -17,7 +17,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import datadog.trace.api.Trace; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.RetryableException; @@ -151,7 +150,7 @@ public JobCreationOutput createNewJob(final JobCreationInput input) { final StandardDestinationDefinition destinationDef = configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId()); - final String destinationImageName = DockerUtils.getTaggedImageName(destinationDef.getDockerRepository(), destinationDef.getDockerImageTag()); + final String destinationImageName = destinationDef.getDockerRepository() + ":" + destinationDef.getDockerImageTag(); final List standardSyncOperations = Lists.newArrayList(); for (final var operationId : standardSync.getOperationIds()) { diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java index bf95d39b8796..a246a2c3faae 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java @@ -10,7 +10,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.temporal.exception.RetryableException; import io.airbyte.commons.version.Version; import io.airbyte.config.AttemptFailureSummary; @@ -125,7 +124,7 @@ class JobCreationAndStatusUpdateActivityTest { private static final UUID DESTINATION_DEFINITION_ID = UUID.randomUUID(); private static final String DOCKER_REPOSITORY = "docker-repo"; private static final String DOCKER_IMAGE_TAG = "0.0.1"; - private static final String DOCKER_IMAGE_NAME = DockerUtils.getTaggedImageName(DOCKER_REPOSITORY, DOCKER_IMAGE_TAG); + private static final String DOCKER_IMAGE_NAME = DOCKER_REPOSITORY + ":" + DOCKER_IMAGE_TAG; private static final Version DESTINATION_PROTOCOL_VERSION = new Version("0.4.0"); private static final long JOB_ID = 123L; private static final long PREVIOUS_JOB_ID = 120L; diff --git a/settings.gradle b/settings.gradle index ebab85551230..6a2522b04182 100644 --- a/settings.gradle +++ b/settings.gradle @@ -82,7 +82,6 @@ include ':airbyte-commons' // shared by CONNECTORS_BASE and PLATFORM sub builds include ':airbyte-api' include ':airbyte-commons-cli' -include ':airbyte-commons-docker' include ':airbyte-commons-protocol' include ':airbyte-config:init' include ':airbyte-config:config-models' // reused by acceptance tests in connector base.