From 4ebf4ecd4df74072ae4646d6c7b7fdb5e4fcc88b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 21 Oct 2022 13:27:49 -0700 Subject: [PATCH 01/34] Tmp --- airbyte-server/build.gradle | 6 ++ .../server/ServerApplicationInitializer.java | 77 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index b38e879fcb5b..ec46beac9ebe 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -8,6 +8,12 @@ configurations.all { } dependencies { + annotationProcessor platform(libs.micronaut.bom) + annotationProcessor libs.bundles.micronaut.annotation.processor + + implementation platform(libs.micronaut.bom) + implementation libs.bundles.micronaut + implementation project(':airbyte-analytics') implementation project(':airbyte-api') implementation project(':airbyte-commons-docker') diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java new file mode 100644 index 000000000000..4c1300e5aa20 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java @@ -0,0 +1,77 @@ +package io.airbyte.server; + +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.server.errors.InvalidInputExceptionMapper; +import io.airbyte.server.errors.InvalidJsonExceptionMapper; +import io.airbyte.server.errors.InvalidJsonInputExceptionMapper; +import io.airbyte.server.errors.KnownExceptionMapper; +import io.airbyte.server.errors.NotFoundExceptionMapper; +import io.airbyte.server.errors.UncaughtExceptionMapper; +import io.micronaut.context.annotation.Requires; +import io.micronaut.context.env.Environment; +import io.micronaut.context.event.ApplicationEventListener; +import io.micronaut.discovery.event.ServiceReadyEvent; +import jakarta.inject.Singleton; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.slf4j.MDC; + +/** + * Performs any required initialization logic on application context start. + */ +@Singleton +@Requires(notEnv = {Environment.TEST}) +@Slf4j +public class ServerApplicationInitializer implements ApplicationEventListener { + + @Override public void onApplicationEvent(final ServiceReadyEvent event) { + final Server server = new Server(PORT); + + final ServletContextHandler handler = new ServletContextHandler(); + + final Map mdc = MDC.getCopyOfContextMap(); + + final ResourceConfig rc = + new ResourceConfig() + .register(new RequestLogger(mdc)) + .register(InvalidInputExceptionMapper.class) + .register(InvalidJsonExceptionMapper.class) + .register(InvalidJsonInputExceptionMapper.class) + .register(KnownExceptionMapper.class) + .register(UncaughtExceptionMapper.class) + .register(NotFoundExceptionMapper.class) + // needed so that the custom json exception mappers don't get overridden + // https://stackoverflow.com/questions/35669774/jersey-custom-exception-mapper-for-invalid-json-string + .register(JacksonJaxbJsonProvider.class); + + // inject custom server functionality + customComponentClasses.forEach(rc::register); + customComponents.forEach(rc::register); + + final ServletHolder configServlet = new ServletHolder(new ServletContainer(rc)); + + handler.addServlet(configServlet, "/api/*"); + + server.setHandler(handler); + + server.start(); + final String banner = MoreResources.readResource("banner/banner.txt"); + LOGGER.info(banner + String.format("Version: %s\n", airbyteVersion.serialize())); + server.join(); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + server.stop(); + } catch (final Exception ex) { + // silently fail at this stage because server is terminating. + LOGGER.warn("exception: " + ex); + } + })); + } +} From d930c98a041d85de0151b980118e9e4aec9a1d26 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 24 Oct 2022 15:46:58 -0700 Subject: [PATCH 02/34] Extract the Attempt API from the V1 API --- airbyte-server/build.gradle | 6 -- .../server/ServerApplicationInitializer.java | 77 ------------------- .../java/io/airbyte/server/ServerFactory.java | 9 ++- .../airbyte/server/apis/AttemptApiImpl.java | 28 +++++++ .../airbyte/server/apis/ConfigurationApi.java | 18 ++--- .../server/apis/binders/AttemptApiBinder.java | 21 +++++ .../apis/factories/AttemptApiFactory.java | 35 +++++++++ 7 files changed, 99 insertions(+), 95 deletions(-) delete mode 100644 airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiImpl.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index ec46beac9ebe..b38e879fcb5b 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -8,12 +8,6 @@ configurations.all { } dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation project(':airbyte-analytics') implementation project(':airbyte-api') implementation project(':airbyte-commons-docker') diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java deleted file mode 100644 index 4c1300e5aa20..000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApplicationInitializer.java +++ /dev/null @@ -1,77 +0,0 @@ -package io.airbyte.server; - -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.server.errors.InvalidInputExceptionMapper; -import io.airbyte.server.errors.InvalidJsonExceptionMapper; -import io.airbyte.server.errors.InvalidJsonInputExceptionMapper; -import io.airbyte.server.errors.KnownExceptionMapper; -import io.airbyte.server.errors.NotFoundExceptionMapper; -import io.airbyte.server.errors.UncaughtExceptionMapper; -import io.micronaut.context.annotation.Requires; -import io.micronaut.context.env.Environment; -import io.micronaut.context.event.ApplicationEventListener; -import io.micronaut.discovery.event.ServiceReadyEvent; -import jakarta.inject.Singleton; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; -import org.glassfish.jersey.server.ResourceConfig; -import org.glassfish.jersey.servlet.ServletContainer; -import org.slf4j.MDC; - -/** - * Performs any required initialization logic on application context start. - */ -@Singleton -@Requires(notEnv = {Environment.TEST}) -@Slf4j -public class ServerApplicationInitializer implements ApplicationEventListener { - - @Override public void onApplicationEvent(final ServiceReadyEvent event) { - final Server server = new Server(PORT); - - final ServletContextHandler handler = new ServletContextHandler(); - - final Map mdc = MDC.getCopyOfContextMap(); - - final ResourceConfig rc = - new ResourceConfig() - .register(new RequestLogger(mdc)) - .register(InvalidInputExceptionMapper.class) - .register(InvalidJsonExceptionMapper.class) - .register(InvalidJsonInputExceptionMapper.class) - .register(KnownExceptionMapper.class) - .register(UncaughtExceptionMapper.class) - .register(NotFoundExceptionMapper.class) - // needed so that the custom json exception mappers don't get overridden - // https://stackoverflow.com/questions/35669774/jersey-custom-exception-mapper-for-invalid-json-string - .register(JacksonJaxbJsonProvider.class); - - // inject custom server functionality - customComponentClasses.forEach(rc::register); - customComponents.forEach(rc::register); - - final ServletHolder configServlet = new ServletHolder(new ServletContainer(rc)); - - handler.addServlet(configServlet, "/api/*"); - - server.setHandler(handler); - - server.start(); - final String banner = MoreResources.readResource("banner/banner.txt"); - LOGGER.info(banner + String.format("Version: %s\n", airbyteVersion.serialize())); - server.join(); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - server.stop(); - } catch (final Exception ex) { - // silently fail at this stage because server is terminating. - LOGGER.warn("exception: " + ex); - } - })); - } -} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index a86486d53961..45a46eccdbfd 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -14,7 +14,10 @@ import io.airbyte.config.persistence.StatePersistence; import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.server.apis.AttemptApiImpl; import io.airbyte.server.apis.ConfigurationApi; +import io.airbyte.server.apis.binders.AttemptApiBinder; +import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; @@ -82,9 +85,11 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul configsFlyway, jobsFlyway); + AttemptApiFactory.setValues(jobPersistence, MDC.getCopyOfContextMap()); + // server configurations - final Set> componentClasses = Set.of(ConfigurationApi.class); - final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder()); + final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiImpl.class); + final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiImpl.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiImpl.java new file mode 100644 index 000000000000..fd1e31573e43 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiImpl.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.AttemptApi; +import io.airbyte.api.model.generated.InternalOperationResult; +import io.airbyte.api.model.generated.SetWorkflowInAttemptRequestBody; +import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.server.handlers.AttemptHandler; +import javax.ws.rs.Path; + +@Path("/v1/attempt/set_workflow_in_attempt") +public class AttemptApiImpl implements AttemptApi { + + private final AttemptHandler attemptHandler; + + public AttemptApiImpl(final JobPersistence jobPersistence) { + attemptHandler = new AttemptHandler(jobPersistence); + } + + @Override + public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody requestBody) { + return ConfigurationApi.execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index ae575aaa2572..8ec915b53978 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -113,7 +113,6 @@ import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.server.errors.BadObjectSchemaKnownException; import io.airbyte.server.errors.IdNotFoundKnownException; -import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.server.handlers.DestinationDefinitionsHandler; @@ -164,7 +163,6 @@ public class ConfigurationApi implements io.airbyte.api.generated.V1Api { private final OpenApiConfigHandler openApiConfigHandler; private final DbMigrationHandler dbMigrationHandler; private final OAuthHandler oAuthHandler; - private final AttemptHandler attemptHandler; private final WorkerEnvironment workerEnvironment; private final LogConfigs logConfigs; private final Path workspaceRoot; @@ -249,7 +247,6 @@ public ConfigurationApi(final ConfigRepository configRepository, logsHandler = new LogsHandler(); openApiConfigHandler = new OpenApiConfigHandler(); dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); - attemptHandler = new AttemptHandler(jobPersistence); } // WORKSPACE @@ -433,6 +430,11 @@ public void setInstancewideSourceOauthParams(final SetInstancewideSourceOauthPar }); } + @Override + public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody setWorkflowInAttemptRequestBody) { + return null; + } + // SOURCE IMPLEMENTATION @Override @@ -835,12 +837,8 @@ public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(final WebBacke return execute(() -> webBackendConnectionsHandler.getWorkspaceState(webBackendWorkspaceState)); } - @Override - public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody requestBody) { - return execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); - } - - private static T execute(final HandlerCall call) { + // TODO: Move to common when all the api are moved + static T execute(final HandlerCall call) { try { return call.call(); } catch (final ConfigNotFoundException e) { @@ -854,7 +852,7 @@ private static T execute(final HandlerCall call) { } } - private interface HandlerCall { + interface HandlerCall { T call() throws ConfigNotFoundException, IOException, JsonValidationException; diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java new file mode 100644 index 000000000000..b49f27e18d77 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.AttemptApiImpl; +import io.airbyte.server.apis.factories.AttemptApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class AttemptApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(AttemptApiFactory.class) + .to(AttemptApiImpl.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java new file mode 100644 index 000000000000..dbba3748306a --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.server.apis.AttemptApiImpl; +import java.util.Map; +import org.glassfish.hk2.api.Factory; +import org.slf4j.MDC; + +public class AttemptApiFactory implements Factory { + + private static JobPersistence jobPersistence; + private static Map mdc; + + public static void setValues(final JobPersistence jobPersistence, final Map mdc) { + AttemptApiFactory.jobPersistence = jobPersistence; + AttemptApiFactory.mdc = mdc; + } + + @Override + public AttemptApiImpl provide() { + MDC.setContextMap(AttemptApiFactory.mdc); + + return new AttemptApiImpl(jobPersistence); + } + + @Override + public void dispose(final AttemptApiImpl instance) { + /* no op */ + } + +} From 2dba5674046e1fda1a5e6a3221b84fff11b77dfc Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 24 Oct 2022 16:06:35 -0700 Subject: [PATCH 03/34] Add comments --- .../main/java/io/airbyte/server/apis/ConfigurationApi.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 8ec915b53978..7e4773b4f365 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -140,6 +140,7 @@ import java.nio.file.Path; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.NotImplementedException; import org.flywaydb.core.Flyway; @javax.ws.rs.Path("/v1") @@ -430,9 +431,13 @@ public void setInstancewideSourceOauthParams(final SetInstancewideSourceOauthPar }); } + /** + * This implementation has been moved to {@link AttemptApiImpl}. Since the path of {@link AttemptApiImpl} is more granular, it will override this + * implementation + */ @Override public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody setWorkflowInAttemptRequestBody) { - return null; + throw new NotImplementedException(); } // SOURCE IMPLEMENTATION From 7c369759b71edbc6c93595b305b6d483d334c500 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 24 Oct 2022 16:49:46 -0700 Subject: [PATCH 04/34] Move Connection API out of configuration API --- .../java/io/airbyte/server/ServerFactory.java | 6 +- .../airbyte/server/apis/ConfigurationApi.java | 76 ++++++-- .../server/apis/ConnectionApiImpl.java | 181 ++++++++++++++++++ .../apis/binders/ConnectionApiBinder.java | 21 ++ .../apis/factories/ConnectionApiFactory.java | 77 ++++++++ 5 files changed, 343 insertions(+), 18 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 45a46eccdbfd..9352ed474268 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -16,7 +16,9 @@ import io.airbyte.persistence.job.JobPersistence; import io.airbyte.server.apis.AttemptApiImpl; import io.airbyte.server.apis.ConfigurationApi; +import io.airbyte.server.apis.ConnectionApiImpl; import io.airbyte.server.apis.binders.AttemptApiBinder; +import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; @@ -88,8 +90,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul AttemptApiFactory.setValues(jobPersistence, MDC.getCopyOfContextMap()); // server configurations - final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiImpl.class); - final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder()); + final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiImpl.class, ConnectionApiImpl.class); + final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 7e4773b4f365..fdea3e07c2ab 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -652,53 +652,85 @@ public CheckConnectionRead checkConnectionToDestinationForUpdate(final Destinati // CONNECTION + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionRead createConnection(final ConnectionCreate connectionCreate) { - return execute(() -> connectionsHandler.createConnection(connectionCreate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) { - return execute(() -> connectionsHandler.updateConnection(connectionUpdate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> connectionsHandler.listAllConnectionsForWorkspace(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) { - return execute(() -> connectionsHandler.searchConnections(connectionSearch)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> connectionsHandler.getConnection(connectionIdRequestBody.getConnectionId())); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - execute(() -> { - operationsHandler.deleteOperationsForConnection(connectionIdRequestBody); - connectionsHandler.deleteConnection(connectionIdRequestBody.getConnectionId()); - return null; - }); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> schedulerHandler.syncConnection(connectionIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public JobInfoRead resetConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> schedulerHandler.resetConnection(connectionIdRequestBody)); + throw new NotImplementedException(); } // Operations @@ -713,9 +745,13 @@ public OperationRead createOperation(final OperationCreate operationCreate) { return execute(() -> operationsHandler.createOperation(operationCreate)); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { - return execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); + throw new NotImplementedException(); } @Override @@ -741,9 +777,13 @@ public OperationRead updateOperation(final OperationUpdate operationUpdate) { return execute(() -> operationsHandler.updateOperation(operationUpdate)); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> stateHandler.getState(connectionIdRequestBody)); + throw new NotImplementedException(); } // SCHEDULER @@ -832,9 +872,13 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne return execute(() -> webBackendConnectionsHandler.webBackendUpdateConnection(webBackendConnectionUpdate)); } + /** + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this + * implementation + */ @Override public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); + throw new NotImplementedException(); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java new file mode 100644 index 000000000000..5c358250ff93 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java @@ -0,0 +1,181 @@ +package io.airbyte.server.apis; + +import io.airbyte.analytics.TrackingClient; +import io.airbyte.api.generated.ConnectionApi; +import io.airbyte.api.model.generated.ConnectionCreate; +import io.airbyte.api.model.generated.ConnectionIdRequestBody; +import io.airbyte.api.model.generated.ConnectionRead; +import io.airbyte.api.model.generated.ConnectionReadList; +import io.airbyte.api.model.generated.ConnectionSearch; +import io.airbyte.api.model.generated.ConnectionState; +import io.airbyte.api.model.generated.ConnectionStateCreateOrUpdate; +import io.airbyte.api.model.generated.ConnectionStateType; +import io.airbyte.api.model.generated.ConnectionUpdate; +import io.airbyte.api.model.generated.JobInfoRead; +import io.airbyte.api.model.generated.WorkspaceIdRequestBody; +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.helpers.LogConfigs; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SecretsRepositoryReader; +import io.airbyte.config.persistence.SecretsRepositoryWriter; +import io.airbyte.config.persistence.StatePersistence; +import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.WorkspaceHelper; +import io.airbyte.server.handlers.ConnectionsHandler; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.JobHistoryHandler; +import io.airbyte.server.handlers.OperationsHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import io.airbyte.server.handlers.SourceDefinitionsHandler; +import io.airbyte.server.handlers.SourceHandler; +import io.airbyte.server.handlers.StateHandler; +import io.airbyte.server.handlers.WebBackendConnectionsHandler; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; +import io.airbyte.validation.json.JsonSchemaValidator; + +public class ConnectionApiImpl implements ConnectionApi { + + private final ConnectionsHandler connectionsHandler; + private final OperationsHandler operationsHandler; + private final SchedulerHandler schedulerHandler; + private final StateHandler stateHandler; + private final WebBackendConnectionsHandler webBackendConnectionsHandler; + + public ConnectionApiImpl(final ConfigRepository configRepository, + final JobPersistence jobPersistence, + final TrackingClient trackingClient, + final EventRunner eventRunner, + final SecretsRepositoryReader secretsRepositoryReader, + final SecretsRepositoryWriter secretsRepositoryWriter, + final SynchronousSchedulerClient synchronousSchedulerClient, + final WorkerEnvironment workerEnvironment, + final LogConfigs logConfigs, + final StatePersistence statePersistence, + final AirbyteVersion airbyteVersion) { + + final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); + + final WorkspaceHelper workspaceHelper = new WorkspaceHelper(configRepository, jobPersistence); + + connectionsHandler = new ConnectionsHandler( + configRepository, + workspaceHelper, + trackingClient, + eventRunner); + + operationsHandler = new OperationsHandler(configRepository); + + schedulerHandler = new SchedulerHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + synchronousSchedulerClient, + jobPersistence, + workerEnvironment, + logConfigs, + eventRunner); + + stateHandler = new StateHandler(statePersistence); + + final SourceHandler sourceHandler = new SourceHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + schemaValidator, + connectionsHandler); + + final DestinationHandler destinationHandler = new DestinationHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + schemaValidator, + connectionsHandler); + + final SourceDefinitionsHandler sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, synchronousSchedulerClient, sourceHandler); + + final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, synchronousSchedulerClient, + destinationHandler); + + final JobHistoryHandler jobHistoryHandler = new JobHistoryHandler(jobPersistence, workerEnvironment, logConfigs, connectionsHandler, sourceHandler, + sourceDefinitionsHandler, destinationHandler, destinationDefinitionsHandler, airbyteVersion); + + webBackendConnectionsHandler = new WebBackendConnectionsHandler( + connectionsHandler, + stateHandler, + sourceHandler, + destinationHandler, + jobHistoryHandler, + schedulerHandler, + operationsHandler, + eventRunner, + configRepository); + } + + @Override + public ConnectionRead createConnection(final ConnectionCreate connectionCreate) { + return ConfigurationApi.execute(() -> connectionsHandler.createConnection(connectionCreate)); + } + + @Override + public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { + return ConfigurationApi.execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); + } + + @Override + public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) { + return ConfigurationApi.execute(() -> connectionsHandler.updateConnection(connectionUpdate)); + } + + @Override + public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)); + } + + @Override + public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> connectionsHandler.listAllConnectionsForWorkspace(workspaceIdRequestBody)); + } + + @Override + public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) { + return ConfigurationApi.execute(() -> connectionsHandler.searchConnections(connectionSearch)); + } + + @Override + public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + return ConfigurationApi.execute(() -> connectionsHandler.getConnection(connectionIdRequestBody.getConnectionId())); + } + + @Override + public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { + return ConfigurationApi.execute(() -> stateHandler.getState(connectionIdRequestBody)); + } + + @Override + public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { + // TODO: move to connectionHandler + return ConfigurationApi.execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); + } + + @Override + public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + ConfigurationApi.execute(() -> { + operationsHandler.deleteOperationsForConnection(connectionIdRequestBody); + connectionsHandler.deleteConnection(connectionIdRequestBody.getConnectionId()); + return null; + }); + } + + @Override + public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + return ConfigurationApi.execute(() -> schedulerHandler.syncConnection(connectionIdRequestBody)); + } + + @Override + public JobInfoRead resetConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + return ConfigurationApi.execute(() -> schedulerHandler.resetConnection(connectionIdRequestBody)); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java new file mode 100644 index 000000000000..540ea185c597 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.ConnectionApiImpl; +import io.airbyte.server.apis.factories.ConnectionApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class ConnectionApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(ConnectionApiFactory.class) + .to(ConnectionApiImpl.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java new file mode 100644 index 000000000000..9b2d929358de --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.analytics.TrackingClient; +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.helpers.LogConfigs; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SecretsRepositoryReader; +import io.airbyte.config.persistence.SecretsRepositoryWriter; +import io.airbyte.config.persistence.StatePersistence; +import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.server.apis.ConnectionApiImpl; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; +import java.util.Map; +import org.glassfish.hk2.api.Factory; +import org.slf4j.MDC; + +public class ConnectionApiFactory implements Factory { + + private static ConfigRepository configRepository; + private static JobPersistence jobPersistence; + private static TrackingClient trackingClient; + private static EventRunner eventRunner; + private static SecretsRepositoryReader secretsRepositoryReader; + private static SecretsRepositoryWriter secretsRepositoryWriter; + private static SynchronousSchedulerClient synchronousSchedulerClient; + private static WorkerEnvironment workerEnvironment; + private static LogConfigs logConfigs; + private static StatePersistence statePersistence; + private static AirbyteVersion airbyteVersion; + private static Map mdc; + + public static void setValues(final ConfigRepository configRepository, + final JobPersistence jobPersistence, + final TrackingClient trackingClient, + final EventRunner eventRunner, + final SecretsRepositoryReader secretsRepositoryReader, + final SecretsRepositoryWriter secretsRepositoryWriter, + final SynchronousSchedulerClient synchronousSchedulerClient, + final WorkerEnvironment workerEnvironment, + final LogConfigs logConfigs, + final StatePersistence statePersistence, + final AirbyteVersion airbyteVersion, + final Map mdc) { + ConnectionApiFactory.configRepository = configRepository; + ConnectionApiFactory.jobPersistence = jobPersistence; + ConnectionApiFactory.trackingClient = trackingClient; + ConnectionApiFactory.eventRunner = eventRunner; + ConnectionApiFactory.secretsRepositoryReader = secretsRepositoryReader; + ConnectionApiFactory.secretsRepositoryWriter = secretsRepositoryWriter; + ConnectionApiFactory.synchronousSchedulerClient = synchronousSchedulerClient; + ConnectionApiFactory.workerEnvironment = workerEnvironment; + ConnectionApiFactory.logConfigs = logConfigs; + ConnectionApiFactory.statePersistence = statePersistence; + ConnectionApiFactory.airbyteVersion = airbyteVersion; + ConnectionApiFactory.mdc = mdc; + } + + @Override + public ConnectionApiImpl provide() { + MDC.setContextMap(ConnectionApiFactory.mdc); + + return new ConnectionApiImpl(configRepository, jobPersistence, trackingClient, eventRunner, secretsRepositoryReader, secretsRepositoryWriter, + synchronousSchedulerClient, workerEnvironment, logConfigs, statePersistence, airbyteVersion); + } + + @Override + public void dispose(final ConnectionApiImpl instance) { + /* no op */ + } + +} From 0d8750993f5e837a91d32558d872c025218e6a49 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 24 Oct 2022 16:50:32 -0700 Subject: [PATCH 05/34] format --- .../main/java/io/airbyte/server/apis/ConfigurationApi.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 7e4773b4f365..e80a2ee5705e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -432,8 +432,8 @@ public void setInstancewideSourceOauthParams(final SetInstancewideSourceOauthPar } /** - * This implementation has been moved to {@link AttemptApiImpl}. Since the path of {@link AttemptApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link AttemptApiImpl}. Since the path of + * {@link AttemptApiImpl} is more granular, it will override this implementation */ @Override public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody setWorkflowInAttemptRequestBody) { From 04e9bb2c1effaf136ee1dbadde986529361fe74e Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 24 Oct 2022 16:53:38 -0700 Subject: [PATCH 06/34] format --- .../airbyte/server/apis/ConfigurationApi.java | 48 +++++++++---------- .../server/apis/ConnectionApiImpl.java | 18 +++++-- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 897b53813273..b98850497f96 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -653,8 +653,8 @@ public CheckConnectionRead checkConnectionToDestinationForUpdate(final Destinati // CONNECTION /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionRead createConnection(final ConnectionCreate connectionCreate) { @@ -662,8 +662,8 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) { @@ -671,8 +671,8 @@ public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { @@ -680,8 +680,8 @@ public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBo } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { @@ -689,8 +689,8 @@ public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdReques } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) { @@ -698,8 +698,8 @@ public ConnectionReadList searchConnections(final ConnectionSearch connectionSea } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -707,8 +707,8 @@ public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRe } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -716,8 +716,8 @@ public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBo } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -725,8 +725,8 @@ public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequ } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public JobInfoRead resetConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -746,8 +746,8 @@ public OperationRead createOperation(final OperationCreate operationCreate) { } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { @@ -778,8 +778,8 @@ public OperationRead updateOperation(final OperationUpdate operationUpdate) { } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -873,8 +873,8 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of {@link ConnectionApiImpl} is more granular, it will override this - * implementation + * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of + * {@link ConnectionApiImpl} is more granular, it will override this implementation */ @Override public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java index 5c358250ff93..fe0c1030d137 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis; import io.airbyte.analytics.TrackingClient; @@ -94,13 +98,16 @@ public ConnectionApiImpl(final ConfigRepository configRepository, schemaValidator, connectionsHandler); - final SourceDefinitionsHandler sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, synchronousSchedulerClient, sourceHandler); + final SourceDefinitionsHandler sourceDefinitionsHandler = + new SourceDefinitionsHandler(configRepository, synchronousSchedulerClient, sourceHandler); - final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, synchronousSchedulerClient, - destinationHandler); + final DestinationDefinitionsHandler destinationDefinitionsHandler = + new DestinationDefinitionsHandler(configRepository, synchronousSchedulerClient, + destinationHandler); - final JobHistoryHandler jobHistoryHandler = new JobHistoryHandler(jobPersistence, workerEnvironment, logConfigs, connectionsHandler, sourceHandler, - sourceDefinitionsHandler, destinationHandler, destinationDefinitionsHandler, airbyteVersion); + final JobHistoryHandler jobHistoryHandler = + new JobHistoryHandler(jobPersistence, workerEnvironment, logConfigs, connectionsHandler, sourceHandler, + sourceDefinitionsHandler, destinationHandler, destinationDefinitionsHandler, airbyteVersion); webBackendConnectionsHandler = new WebBackendConnectionsHandler( connectionsHandler, @@ -178,4 +185,5 @@ public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequ public JobInfoRead resetConnection(final ConnectionIdRequestBody connectionIdRequestBody) { return ConfigurationApi.execute(() -> schedulerHandler.resetConnection(connectionIdRequestBody)); } + } From c74db48ea87d654dd94df5e5df79d33349b8fe8e Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 08:19:55 -0700 Subject: [PATCH 07/34] Rename to Controller --- .../src/main/java/io/airbyte/server/ServerFactory.java | 4 ++-- .../{AttemptApiImpl.java => AttemptApiController.java} | 4 ++-- .../java/io/airbyte/server/apis/ConfigurationApi.java | 4 ++-- .../airbyte/server/apis/binders/AttemptApiBinder.java | 4 ++-- .../server/apis/factories/AttemptApiFactory.java | 10 +++++----- 5 files changed, 13 insertions(+), 13 deletions(-) rename airbyte-server/src/main/java/io/airbyte/server/apis/{AttemptApiImpl.java => AttemptApiController.java} (85%) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 45a46eccdbfd..f943a52419d4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -14,7 +14,7 @@ import io.airbyte.config.persistence.StatePersistence; import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.server.apis.AttemptApiImpl; +import io.airbyte.server.apis.AttemptApiController; import io.airbyte.server.apis.ConfigurationApi; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; @@ -88,7 +88,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul AttemptApiFactory.setValues(jobPersistence, MDC.getCopyOfContextMap()); // server configurations - final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiImpl.class); + final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class); final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder()); // construct server diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiImpl.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java similarity index 85% rename from airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiImpl.java rename to airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java index fd1e31573e43..4aa363a9a22c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiImpl.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java @@ -12,11 +12,11 @@ import javax.ws.rs.Path; @Path("/v1/attempt/set_workflow_in_attempt") -public class AttemptApiImpl implements AttemptApi { +public class AttemptApiController implements AttemptApi { private final AttemptHandler attemptHandler; - public AttemptApiImpl(final JobPersistence jobPersistence) { + public AttemptApiController(final JobPersistence jobPersistence) { attemptHandler = new AttemptHandler(jobPersistence); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index e80a2ee5705e..81fc1daff33b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -432,8 +432,8 @@ public void setInstancewideSourceOauthParams(final SetInstancewideSourceOauthPar } /** - * This implementation has been moved to {@link AttemptApiImpl}. Since the path of - * {@link AttemptApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link AttemptApiController}. Since the path of + * {@link AttemptApiController} is more granular, it will override this implementation */ @Override public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody setWorkflowInAttemptRequestBody) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java index b49f27e18d77..2eb09dddaf02 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java @@ -4,7 +4,7 @@ package io.airbyte.server.apis.binders; -import io.airbyte.server.apis.AttemptApiImpl; +import io.airbyte.server.apis.AttemptApiController; import io.airbyte.server.apis.factories.AttemptApiFactory; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.process.internal.RequestScoped; @@ -14,7 +14,7 @@ public class AttemptApiBinder extends AbstractBinder { @Override protected void configure() { bindFactory(AttemptApiFactory.class) - .to(AttemptApiImpl.class) + .to(AttemptApiController.class) .in(RequestScoped.class); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java index dbba3748306a..9fea2dda023c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java @@ -5,12 +5,12 @@ package io.airbyte.server.apis.factories; import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.server.apis.AttemptApiImpl; +import io.airbyte.server.apis.AttemptApiController; import java.util.Map; import org.glassfish.hk2.api.Factory; import org.slf4j.MDC; -public class AttemptApiFactory implements Factory { +public class AttemptApiFactory implements Factory { private static JobPersistence jobPersistence; private static Map mdc; @@ -21,14 +21,14 @@ public static void setValues(final JobPersistence jobPersistence, final Map Date: Tue, 25 Oct 2022 08:31:25 -0700 Subject: [PATCH 08/34] Rename to Controller --- .../java/io/airbyte/server/ServerFactory.java | 4 +- .../airbyte/server/apis/ConfigurationApi.java | 48 +++++++++---------- ...Impl.java => ConnectionApiController.java} | 24 +++++----- .../apis/binders/ConnectionApiBinder.java | 4 +- .../apis/factories/ConnectionApiFactory.java | 10 ++-- 5 files changed, 45 insertions(+), 45 deletions(-) rename airbyte-server/src/main/java/io/airbyte/server/apis/{ConnectionApiImpl.java => ConnectionApiController.java} (88%) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 4c84abb2d4a2..be8d6fe93cdc 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -16,7 +16,7 @@ import io.airbyte.persistence.job.JobPersistence; import io.airbyte.server.apis.AttemptApiController; import io.airbyte.server.apis.ConfigurationApi; -import io.airbyte.server.apis.ConnectionApiImpl; +import io.airbyte.server.apis.ConnectionApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; @@ -90,7 +90,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul AttemptApiFactory.setValues(jobPersistence, MDC.getCopyOfContextMap()); // server configurations - final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiImpl.class); + final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class); final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder()); // construct server diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 1ba590f6b1be..2650e8a1842b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -653,8 +653,8 @@ public CheckConnectionRead checkConnectionToDestinationForUpdate(final Destinati // CONNECTION /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionRead createConnection(final ConnectionCreate connectionCreate) { @@ -662,8 +662,8 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) { @@ -671,8 +671,8 @@ public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { @@ -680,8 +680,8 @@ public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBo } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { @@ -689,8 +689,8 @@ public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdReques } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) { @@ -698,8 +698,8 @@ public ConnectionReadList searchConnections(final ConnectionSearch connectionSea } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -707,8 +707,8 @@ public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRe } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -716,8 +716,8 @@ public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBo } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -725,8 +725,8 @@ public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequ } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public JobInfoRead resetConnection(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -746,8 +746,8 @@ public OperationRead createOperation(final OperationCreate operationCreate) { } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { @@ -778,8 +778,8 @@ public OperationRead updateOperation(final OperationUpdate operationUpdate) { } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { @@ -873,8 +873,8 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne } /** - * This implementation has been moved to {@link ConnectionApiImpl}. Since the path of - * {@link ConnectionApiImpl} is more granular, it will override this implementation + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation */ @Override public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java similarity index 88% rename from airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java rename to airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index fe0c1030d137..d02bc0f2b06c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiImpl.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -40,7 +40,7 @@ import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.validation.json.JsonSchemaValidator; -public class ConnectionApiImpl implements ConnectionApi { +public class ConnectionApiController implements ConnectionApi { private final ConnectionsHandler connectionsHandler; private final OperationsHandler operationsHandler; @@ -48,17 +48,17 @@ public class ConnectionApiImpl implements ConnectionApi { private final StateHandler stateHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; - public ConnectionApiImpl(final ConfigRepository configRepository, - final JobPersistence jobPersistence, - final TrackingClient trackingClient, - final EventRunner eventRunner, - final SecretsRepositoryReader secretsRepositoryReader, - final SecretsRepositoryWriter secretsRepositoryWriter, - final SynchronousSchedulerClient synchronousSchedulerClient, - final WorkerEnvironment workerEnvironment, - final LogConfigs logConfigs, - final StatePersistence statePersistence, - final AirbyteVersion airbyteVersion) { + public ConnectionApiController(final ConfigRepository configRepository, + final JobPersistence jobPersistence, + final TrackingClient trackingClient, + final EventRunner eventRunner, + final SecretsRepositoryReader secretsRepositoryReader, + final SecretsRepositoryWriter secretsRepositoryWriter, + final SynchronousSchedulerClient synchronousSchedulerClient, + final WorkerEnvironment workerEnvironment, + final LogConfigs logConfigs, + final StatePersistence statePersistence, + final AirbyteVersion airbyteVersion) { final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java index 540ea185c597..d95fe3385514 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java @@ -4,7 +4,7 @@ package io.airbyte.server.apis.binders; -import io.airbyte.server.apis.ConnectionApiImpl; +import io.airbyte.server.apis.ConnectionApiController; import io.airbyte.server.apis.factories.ConnectionApiFactory; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.process.internal.RequestScoped; @@ -14,7 +14,7 @@ public class ConnectionApiBinder extends AbstractBinder { @Override protected void configure() { bindFactory(ConnectionApiFactory.class) - .to(ConnectionApiImpl.class) + .to(ConnectionApiController.class) .in(RequestScoped.class); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java index 9b2d929358de..0bfd7e440515 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java @@ -13,14 +13,14 @@ import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.config.persistence.StatePersistence; import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.server.apis.ConnectionApiImpl; +import io.airbyte.server.apis.ConnectionApiController; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.util.Map; import org.glassfish.hk2.api.Factory; import org.slf4j.MDC; -public class ConnectionApiFactory implements Factory { +public class ConnectionApiFactory implements Factory { private static ConfigRepository configRepository; private static JobPersistence jobPersistence; @@ -62,15 +62,15 @@ public static void setValues(final ConfigRepository configRepository, } @Override - public ConnectionApiImpl provide() { + public ConnectionApiController provide() { MDC.setContextMap(ConnectionApiFactory.mdc); - return new ConnectionApiImpl(configRepository, jobPersistence, trackingClient, eventRunner, secretsRepositoryReader, secretsRepositoryWriter, + return new ConnectionApiController(configRepository, jobPersistence, trackingClient, eventRunner, secretsRepositoryReader, secretsRepositoryWriter, synchronousSchedulerClient, workerEnvironment, logConfigs, statePersistence, airbyteVersion); } @Override - public void dispose(final ConnectionApiImpl instance) { + public void dispose(final ConnectionApiController instance) { /* no op */ } From 7211da60643c4d6bc3d80904d65289b066e9d7cf Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 08:55:34 -0700 Subject: [PATCH 09/34] Add values to the factory --- .../java/io/airbyte/server/ServerFactory.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index be8d6fe93cdc..d519f5d7f0cd 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -20,6 +20,7 @@ import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; +import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; @@ -89,6 +90,20 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul AttemptApiFactory.setValues(jobPersistence, MDC.getCopyOfContextMap()); + ConnectionApiFactory.setValues( + configRepository, + jobPersistence, + trackingClient, + eventRunner, + secretsRepositoryReader, + secretsRepositoryWriter, + synchronousSchedulerClient, + workerEnvironment, + logConfigs, + new StatePersistence(configsDatabase), + airbyteVersion, + MDC.getCopyOfContextMap()); + // server configurations final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class); final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder()); From f503b4cfc8430d0b23b1ea5d9ea74e9af333a6d6 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 11:06:14 -0700 Subject: [PATCH 10/34] Change the constructor to use hadler instead of objects needed by the handler --- .../java/io/airbyte/server/ServerApp.java | 93 ++++++++++++++++- .../java/io/airbyte/server/ServerFactory.java | 77 +++++++++------ .../server/apis/AttemptApiController.java | 5 +- .../server/apis/ConnectionApiController.java | 99 ++----------------- .../apis/factories/AttemptApiFactory.java | 10 +- .../apis/factories/ConnectionApiFactory.java | 67 ++++--------- 6 files changed, 176 insertions(+), 175 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index ee59c365d776..f3a9ec0409dc 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -27,6 +27,7 @@ import io.airbyte.config.persistence.DatabaseConfigPersistence; import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; +import io.airbyte.config.persistence.StatePersistence; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretPersistence; @@ -42,6 +43,7 @@ import io.airbyte.persistence.job.DefaultJobPersistence; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.WebUrlHelper; +import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.errorreporter.JobErrorReporter; import io.airbyte.persistence.job.errorreporter.JobErrorReportingClient; import io.airbyte.persistence.job.errorreporter.JobErrorReportingClientFactory; @@ -53,10 +55,22 @@ import io.airbyte.server.errors.KnownExceptionMapper; import io.airbyte.server.errors.NotFoundExceptionMapper; import io.airbyte.server.errors.UncaughtExceptionMapper; +import io.airbyte.server.handlers.AttemptHandler; +import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.JobHistoryHandler; +import io.airbyte.server.handlers.OperationsHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import io.airbyte.server.handlers.SourceDefinitionsHandler; +import io.airbyte.server.handlers.SourceHandler; +import io.airbyte.server.handlers.StateHandler; +import io.airbyte.server.handlers.WebBackendConnectionsHandler; import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.TemporalEventRunner; +import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -255,6 +269,77 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, // "major" version bump as it will no longer be needed. migrateExistingConnectionsToTemporalScheduler(configRepository, jobPersistence, eventRunner); + final WorkspaceHelper workspaceHelper = new WorkspaceHelper(configRepository, jobPersistence); + + final AttemptHandler attemptHandler = new AttemptHandler(jobPersistence); + + final ConnectionsHandler connectionsHandler = new ConnectionsHandler( + configRepository, + workspaceHelper, + trackingClient, + eventRunner); + + final OperationsHandler operationsHandler = new OperationsHandler(configRepository); + + final SchedulerHandler schedulerHandler = new SchedulerHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + syncSchedulerClient, + jobPersistence, + configs.getWorkerEnvironment(), + configs.getLogConfigs(), + eventRunner); + + final StatePersistence statePersistence = new StatePersistence(configsDatabase); + + final StateHandler stateHandler = new StateHandler(statePersistence); + + final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); + + final SourceHandler sourceHandler = new SourceHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + schemaValidator, + connectionsHandler); + + final DestinationHandler destinationHandler = new DestinationHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + schemaValidator, + connectionsHandler); + + final SourceDefinitionsHandler sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, syncSchedulerClient, sourceHandler); + + final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler( + configRepository, + syncSchedulerClient, + destinationHandler); + + final JobHistoryHandler jobHistoryHandler = new JobHistoryHandler( + jobPersistence, + configs.getWorkerEnvironment(), + configs.getLogConfigs(), + connectionsHandler, + sourceHandler, + sourceDefinitionsHandler, + destinationHandler, + destinationDefinitionsHandler, + configs.getAirbyteVersion()); + + final WebBackendConnectionsHandler webBackendConnectionsHandler = new WebBackendConnectionsHandler( + connectionsHandler, + stateHandler, + sourceHandler, + destinationHandler, + jobHistoryHandler, + schedulerHandler, + operationsHandler, + eventRunner, + configRepository); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -273,7 +358,13 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, httpClient, eventRunner, configsFlyway, - jobsFlyway); + jobsFlyway, + attemptHandler, + connectionsHandler, + operationsHandler, + schedulerHandler, + stateHandler, + webBackendConnectionsHandler); } @VisibleForTesting diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index d519f5d7f0cd..6bced7164b34 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -21,32 +21,45 @@ import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; +import io.airbyte.server.handlers.AttemptHandler; +import io.airbyte.server.handlers.ConnectionsHandler; +import io.airbyte.server.handlers.OperationsHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import io.airbyte.server.handlers.StateHandler; +import io.airbyte.server.handlers.WebBackendConnectionsHandler; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; import java.nio.file.Path; +import java.util.Map; import java.util.Set; import org.flywaydb.core.Flyway; import org.slf4j.MDC; public interface ServerFactory { - ServerRunnable create(SynchronousSchedulerClient cachingSchedulerClient, - ConfigRepository configRepository, - SecretsRepositoryReader secretsRepositoryReader, - SecretsRepositoryWriter secretsRepositoryWriter, - JobPersistence jobPersistence, - Database configsDatabase, - Database jobsDatabase, - TrackingClient trackingClient, - WorkerEnvironment workerEnvironment, - LogConfigs logConfigs, - AirbyteVersion airbyteVersion, - Path workspaceRoot, - HttpClient httpClient, - EventRunner eventRunner, - Flyway configsFlyway, - Flyway jobsFlyway); + ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClient, + final ConfigRepository configRepository, + final SecretsRepositoryReader secretsRepositoryReader, + final SecretsRepositoryWriter secretsRepositoryWriter, + final JobPersistence jobPersistence, + final Database configsDatabase, + final Database jobsDatabase, + final TrackingClient trackingClient, + final WorkerEnvironment workerEnvironment, + final LogConfigs logConfigs, + final AirbyteVersion airbyteVersion, + final Path workspaceRoot, + final HttpClient httpClient, + final EventRunner eventRunner, + final Flyway configsFlyway, + final Flyway jobsFlyway, + final AttemptHandler attemptHandler, + final ConnectionsHandler connectionsHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler, + final StateHandler stateHandler, + final WebBackendConnectionsHandler webBackendConnectionsHandler); class Api implements ServerFactory { @@ -66,7 +79,15 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final HttpClient httpClient, final EventRunner eventRunner, final Flyway configsFlyway, - final Flyway jobsFlyway) { + final Flyway jobsFlyway, + final AttemptHandler attemptHandler, + final ConnectionsHandler connectionsHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler, + final StateHandler stateHandler, + final WebBackendConnectionsHandler webBackendConnectionsHandler) { + final Map mdc = MDC.getCopyOfContextMap(); + // set static values for factory ConfigurationApiFactory.setValues( configRepository, @@ -75,7 +96,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul jobPersistence, synchronousSchedulerClient, new StatePersistence(configsDatabase), - MDC.getCopyOfContextMap(), + mdc, configsDatabase, jobsDatabase, trackingClient, @@ -88,21 +109,15 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul configsFlyway, jobsFlyway); - AttemptApiFactory.setValues(jobPersistence, MDC.getCopyOfContextMap()); + AttemptApiFactory.setValues(attemptHandler, mdc); ConnectionApiFactory.setValues( - configRepository, - jobPersistence, - trackingClient, - eventRunner, - secretsRepositoryReader, - secretsRepositoryWriter, - synchronousSchedulerClient, - workerEnvironment, - logConfigs, - new StatePersistence(configsDatabase), - airbyteVersion, - MDC.getCopyOfContextMap()); + connectionsHandler, + operationsHandler, + schedulerHandler, + stateHandler, + webBackendConnectionsHandler, + mdc); // server configurations final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java index 4aa363a9a22c..71274154cca4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java @@ -7,7 +7,6 @@ import io.airbyte.api.generated.AttemptApi; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.SetWorkflowInAttemptRequestBody; -import io.airbyte.persistence.job.JobPersistence; import io.airbyte.server.handlers.AttemptHandler; import javax.ws.rs.Path; @@ -16,8 +15,8 @@ public class AttemptApiController implements AttemptApi { private final AttemptHandler attemptHandler; - public AttemptApiController(final JobPersistence jobPersistence) { - attemptHandler = new AttemptHandler(jobPersistence); + public AttemptApiController(final AttemptHandler attemptHandler) { + this.attemptHandler = attemptHandler; } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index d02bc0f2b06c..5f5e8e477730 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -4,7 +4,6 @@ package io.airbyte.server.apis; -import io.airbyte.analytics.TrackingClient; import io.airbyte.api.generated.ConnectionApi; import io.airbyte.api.model.generated.ConnectionCreate; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -17,28 +16,11 @@ import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.JobInfoRead; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.version.AirbyteVersion; -import io.airbyte.config.Configs.WorkerEnvironment; -import io.airbyte.config.helpers.LogConfigs; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.SecretsRepositoryReader; -import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.config.persistence.StatePersistence; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.server.handlers.ConnectionsHandler; -import io.airbyte.server.handlers.DestinationDefinitionsHandler; -import io.airbyte.server.handlers.DestinationHandler; -import io.airbyte.server.handlers.JobHistoryHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; -import io.airbyte.server.handlers.SourceDefinitionsHandler; -import io.airbyte.server.handlers.SourceHandler; import io.airbyte.server.handlers.StateHandler; import io.airbyte.server.handlers.WebBackendConnectionsHandler; -import io.airbyte.server.scheduler.EventRunner; -import io.airbyte.server.scheduler.SynchronousSchedulerClient; -import io.airbyte.validation.json.JsonSchemaValidator; public class ConnectionApiController implements ConnectionApi { @@ -48,77 +30,16 @@ public class ConnectionApiController implements ConnectionApi { private final StateHandler stateHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; - public ConnectionApiController(final ConfigRepository configRepository, - final JobPersistence jobPersistence, - final TrackingClient trackingClient, - final EventRunner eventRunner, - final SecretsRepositoryReader secretsRepositoryReader, - final SecretsRepositoryWriter secretsRepositoryWriter, - final SynchronousSchedulerClient synchronousSchedulerClient, - final WorkerEnvironment workerEnvironment, - final LogConfigs logConfigs, - final StatePersistence statePersistence, - final AirbyteVersion airbyteVersion) { - - final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); - - final WorkspaceHelper workspaceHelper = new WorkspaceHelper(configRepository, jobPersistence); - - connectionsHandler = new ConnectionsHandler( - configRepository, - workspaceHelper, - trackingClient, - eventRunner); - - operationsHandler = new OperationsHandler(configRepository); - - schedulerHandler = new SchedulerHandler( - configRepository, - secretsRepositoryReader, - secretsRepositoryWriter, - synchronousSchedulerClient, - jobPersistence, - workerEnvironment, - logConfigs, - eventRunner); - - stateHandler = new StateHandler(statePersistence); - - final SourceHandler sourceHandler = new SourceHandler( - configRepository, - secretsRepositoryReader, - secretsRepositoryWriter, - schemaValidator, - connectionsHandler); - - final DestinationHandler destinationHandler = new DestinationHandler( - configRepository, - secretsRepositoryReader, - secretsRepositoryWriter, - schemaValidator, - connectionsHandler); - - final SourceDefinitionsHandler sourceDefinitionsHandler = - new SourceDefinitionsHandler(configRepository, synchronousSchedulerClient, sourceHandler); - - final DestinationDefinitionsHandler destinationDefinitionsHandler = - new DestinationDefinitionsHandler(configRepository, synchronousSchedulerClient, - destinationHandler); - - final JobHistoryHandler jobHistoryHandler = - new JobHistoryHandler(jobPersistence, workerEnvironment, logConfigs, connectionsHandler, sourceHandler, - sourceDefinitionsHandler, destinationHandler, destinationDefinitionsHandler, airbyteVersion); - - webBackendConnectionsHandler = new WebBackendConnectionsHandler( - connectionsHandler, - stateHandler, - sourceHandler, - destinationHandler, - jobHistoryHandler, - schedulerHandler, - operationsHandler, - eventRunner, - configRepository); + public ConnectionApiController(final ConnectionsHandler connectionsHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler, + final StateHandler stateHandler, + final WebBackendConnectionsHandler webBackendConnectionsHandler) { + this.connectionsHandler = connectionsHandler; + this.operationsHandler = operationsHandler; + this.schedulerHandler = schedulerHandler; + this.stateHandler = stateHandler; + this.webBackendConnectionsHandler = webBackendConnectionsHandler; } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java index 9fea2dda023c..27fb62696b41 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java @@ -4,19 +4,19 @@ package io.airbyte.server.apis.factories; -import io.airbyte.persistence.job.JobPersistence; import io.airbyte.server.apis.AttemptApiController; +import io.airbyte.server.handlers.AttemptHandler; import java.util.Map; import org.glassfish.hk2.api.Factory; import org.slf4j.MDC; public class AttemptApiFactory implements Factory { - private static JobPersistence jobPersistence; + private static AttemptHandler attemptHandler; private static Map mdc; - public static void setValues(final JobPersistence jobPersistence, final Map mdc) { - AttemptApiFactory.jobPersistence = jobPersistence; + public static void setValues(final AttemptHandler attemptHandler, final Map mdc) { + AttemptApiFactory.attemptHandler = attemptHandler; AttemptApiFactory.mdc = mdc; } @@ -24,7 +24,7 @@ public static void setValues(final JobPersistence jobPersistence, final Map { - private static ConfigRepository configRepository; - private static JobPersistence jobPersistence; - private static TrackingClient trackingClient; - private static EventRunner eventRunner; - private static SecretsRepositoryReader secretsRepositoryReader; - private static SecretsRepositoryWriter secretsRepositoryWriter; - private static SynchronousSchedulerClient synchronousSchedulerClient; - private static WorkerEnvironment workerEnvironment; - private static LogConfigs logConfigs; - private static StatePersistence statePersistence; - private static AirbyteVersion airbyteVersion; + private static ConnectionsHandler connectionsHandler; + private static OperationsHandler operationsHandler; + private static SchedulerHandler schedulerHandler; + private static StateHandler stateHandler; + private static WebBackendConnectionsHandler webBackendConnectionsHandler; private static Map mdc; - public static void setValues(final ConfigRepository configRepository, - final JobPersistence jobPersistence, - final TrackingClient trackingClient, - final EventRunner eventRunner, - final SecretsRepositoryReader secretsRepositoryReader, - final SecretsRepositoryWriter secretsRepositoryWriter, - final SynchronousSchedulerClient synchronousSchedulerClient, - final WorkerEnvironment workerEnvironment, - final LogConfigs logConfigs, - final StatePersistence statePersistence, - final AirbyteVersion airbyteVersion, + public static void setValues(final ConnectionsHandler connectionsHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler, + final StateHandler stateHandler, + final WebBackendConnectionsHandler webBackendConnectionsHandler, final Map mdc) { - ConnectionApiFactory.configRepository = configRepository; - ConnectionApiFactory.jobPersistence = jobPersistence; - ConnectionApiFactory.trackingClient = trackingClient; - ConnectionApiFactory.eventRunner = eventRunner; - ConnectionApiFactory.secretsRepositoryReader = secretsRepositoryReader; - ConnectionApiFactory.secretsRepositoryWriter = secretsRepositoryWriter; - ConnectionApiFactory.synchronousSchedulerClient = synchronousSchedulerClient; - ConnectionApiFactory.workerEnvironment = workerEnvironment; - ConnectionApiFactory.logConfigs = logConfigs; - ConnectionApiFactory.statePersistence = statePersistence; - ConnectionApiFactory.airbyteVersion = airbyteVersion; + ConnectionApiFactory.connectionsHandler = connectionsHandler; + ConnectionApiFactory.operationsHandler = operationsHandler; + ConnectionApiFactory.schedulerHandler = schedulerHandler; + ConnectionApiFactory.stateHandler = stateHandler; + ConnectionApiFactory.webBackendConnectionsHandler = webBackendConnectionsHandler; ConnectionApiFactory.mdc = mdc; } @@ -65,8 +41,7 @@ public static void setValues(final ConfigRepository configRepository, public ConnectionApiController provide() { MDC.setContextMap(ConnectionApiFactory.mdc); - return new ConnectionApiController(configRepository, jobPersistence, trackingClient, eventRunner, secretsRepositoryReader, secretsRepositoryWriter, - synchronousSchedulerClient, workerEnvironment, logConfigs, statePersistence, airbyteVersion); + return new ConnectionApiController(connectionsHandler, operationsHandler, schedulerHandler, stateHandler, webBackendConnectionsHandler); } @Override From 2103e7d9facdf94b50b57754c631865726b1939a Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 15:07:06 -0700 Subject: [PATCH 11/34] Update with new tags. --- .../airbyte/server/apis/ConfigurationApi.java | 18 +++--------------- .../server/apis/ConnectionApiController.java | 19 ------------------- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 88fd32d02203..161a209bd8ad 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -746,13 +746,9 @@ public OperationRead createOperation(final OperationCreate operationCreate) { return execute(() -> operationsHandler.createOperation(operationCreate)); } - /** - * This implementation has been moved to {@link ConnectionApiController}. Since the path of - * {@link ConnectionApiController} is more granular, it will override this implementation - */ @Override public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { - throw new NotImplementedException(); + return ConfigurationApi.execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); } @Override @@ -778,13 +774,9 @@ public OperationRead updateOperation(final OperationUpdate operationUpdate) { return execute(() -> operationsHandler.updateOperation(operationUpdate)); } - /** - * This implementation has been moved to {@link ConnectionApiController}. Since the path of - * {@link ConnectionApiController} is more granular, it will override this implementation - */ @Override public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { - throw new NotImplementedException(); + return ConfigurationApi.execute(() -> stateHandler.getState(connectionIdRequestBody)); } // SCHEDULER @@ -878,13 +870,9 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne return execute(() -> webBackendConnectionsHandler.webBackendUpdateConnection(webBackendConnectionUpdate)); } - /** - * This implementation has been moved to {@link ConnectionApiController}. Since the path of - * {@link ConnectionApiController} is more granular, it will override this implementation - */ @Override public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { - throw new NotImplementedException(); + return ConfigurationApi.execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index 5f5e8e477730..0d74d2a78c3b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -10,9 +10,6 @@ import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.ConnectionReadList; import io.airbyte.api.model.generated.ConnectionSearch; -import io.airbyte.api.model.generated.ConnectionState; -import io.airbyte.api.model.generated.ConnectionStateCreateOrUpdate; -import io.airbyte.api.model.generated.ConnectionStateType; import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.JobInfoRead; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; @@ -47,11 +44,6 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) return ConfigurationApi.execute(() -> connectionsHandler.createConnection(connectionCreate)); } - @Override - public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { - return ConfigurationApi.execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); - } - @Override public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) { return ConfigurationApi.execute(() -> connectionsHandler.updateConnection(connectionUpdate)); @@ -77,17 +69,6 @@ public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRe return ConfigurationApi.execute(() -> connectionsHandler.getConnection(connectionIdRequestBody.getConnectionId())); } - @Override - public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { - return ConfigurationApi.execute(() -> stateHandler.getState(connectionIdRequestBody)); - } - - @Override - public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { - // TODO: move to connectionHandler - return ConfigurationApi.execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); - } - @Override public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBody) { ConfigurationApi.execute(() -> { From a5c852293ce50e9943799bd7e1964690de5a9679 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 15:19:24 -0700 Subject: [PATCH 12/34] tmp --- .../airbyte/server/apis/ConfigurationApi.java | 15 +++++++---- .../server/apis/DbMigrationApiController.java | 26 +++++++++++++++++++ .../apis/factories/DbMigrationApiFactory.java | 25 ++++++++++++++++++ 3 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 161a209bd8ad..f6b202b18d36 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -115,7 +115,6 @@ import io.airbyte.server.errors.BadObjectSchemaKnownException; import io.airbyte.server.errors.IdNotFoundKnownException; import io.airbyte.server.handlers.ConnectionsHandler; -import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.HealthCheckHandler; @@ -163,7 +162,6 @@ public class ConfigurationApi implements io.airbyte.api.generated.V1Api { private final HealthCheckHandler healthCheckHandler; private final LogsHandler logsHandler; private final OpenApiConfigHandler openApiConfigHandler; - private final DbMigrationHandler dbMigrationHandler; private final OAuthHandler oAuthHandler; private final WorkerEnvironment workerEnvironment; private final LogConfigs logConfigs; @@ -248,7 +246,6 @@ public ConfigurationApi(final ConfigRepository configRepository, healthCheckHandler = new HealthCheckHandler(configRepository); logsHandler = new LogsHandler(); openApiConfigHandler = new OpenApiConfigHandler(); - dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); } // WORKSPACE @@ -498,14 +495,22 @@ public SourceDiscoverSchemaRead discoverSchemaForSource(final SourceDiscoverSche // DB MIGRATION + /** + * This implementation has been moved to {@link DbMigrationApiController}. Since the path of + * {@link DbMigrationApiController} is more granular, it will override this implementation + */ @Override public DbMigrationReadList listMigrations(final DbMigrationRequestBody request) { - return execute(() -> dbMigrationHandler.list(request)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DbMigrationApiController}. Since the path of + * {@link DbMigrationApiController} is more granular, it will override this implementation + */ @Override public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody request) { - return execute(() -> dbMigrationHandler.migrate(request)); + throw new NotImplementedException(); } // DESTINATION diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java new file mode 100644 index 000000000000..bba8786c9d56 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java @@ -0,0 +1,26 @@ +package io.airbyte.server.apis; + +import io.airbyte.api.generated.DbMigrationApi; +import io.airbyte.api.model.generated.DbMigrationExecutionRead; +import io.airbyte.api.model.generated.DbMigrationReadList; +import io.airbyte.api.model.generated.DbMigrationRequestBody; +import io.airbyte.server.handlers.DbMigrationHandler; +import javax.ws.rs.Path; + +@Path("/v1/db_migrations") +public class DbMigrationApiController implements DbMigrationApi { + + private final DbMigrationHandler dbMigrationHandler; + + public DbMigrationApiController(final DbMigrationHandler dbMigrationHandler) { + this.dbMigrationHandler = dbMigrationHandler; + } + + @Override public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { + return ConfigurationApi.execute(() -> dbMigrationHandler.migrate(dbMigrationRequestBody)); + } + + @Override public DbMigrationReadList listMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { + return ConfigurationApi.execute(() -> dbMigrationHandler.list(dbMigrationRequestBody)); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java new file mode 100644 index 000000000000..d1b36e6ca5e7 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java @@ -0,0 +1,25 @@ +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.DbMigrationApiController; +import io.airbyte.server.handlers.DbMigrationHandler; +import java.util.Map; +import org.glassfish.hk2.api.Factory; + +public class DbMigrationApiFactory implements Factory { + + private static DbMigrationHandler dbMigrationHandler; + private static Map mdc; + + public static void setValues(final DbMigrationHandler dbMigrationHandler, final Map mdc) { + DbMigrationApiFactory.dbMigrationHandler = dbMigrationHandler; + DbMigrationApiFactory.mdc = mdc; + } + + @Override public DbMigrationApiController provide() { + return null; + } + + @Override public void dispose(final DbMigrationApiController instance) { + /* no op */ + } +} From c5aee8f515ec74387e0f6a526d9d70fa64af1c44 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 15:28:06 -0700 Subject: [PATCH 13/34] Fix PMD errors --- .../java/io/airbyte/server/ServerApp.java | 62 +------------------ .../java/io/airbyte/server/ServerFactory.java | 12 +--- .../server/apis/ConnectionApiController.java | 10 +-- .../apis/factories/ConnectionApiFactory.java | 10 +-- 4 files changed, 5 insertions(+), 89 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index f3a9ec0409dc..74249b560412 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -27,7 +27,6 @@ import io.airbyte.config.persistence.DatabaseConfigPersistence; import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.config.persistence.StatePersistence; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretPersistence; @@ -58,19 +57,11 @@ import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; -import io.airbyte.server.handlers.DestinationDefinitionsHandler; -import io.airbyte.server.handlers.DestinationHandler; -import io.airbyte.server.handlers.JobHistoryHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; -import io.airbyte.server.handlers.SourceDefinitionsHandler; -import io.airbyte.server.handlers.SourceHandler; -import io.airbyte.server.handlers.StateHandler; -import io.airbyte.server.handlers.WebBackendConnectionsHandler; import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.TemporalEventRunner; -import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -291,55 +282,6 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, configs.getLogConfigs(), eventRunner); - final StatePersistence statePersistence = new StatePersistence(configsDatabase); - - final StateHandler stateHandler = new StateHandler(statePersistence); - - final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); - - final SourceHandler sourceHandler = new SourceHandler( - configRepository, - secretsRepositoryReader, - secretsRepositoryWriter, - schemaValidator, - connectionsHandler); - - final DestinationHandler destinationHandler = new DestinationHandler( - configRepository, - secretsRepositoryReader, - secretsRepositoryWriter, - schemaValidator, - connectionsHandler); - - final SourceDefinitionsHandler sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, syncSchedulerClient, sourceHandler); - - final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler( - configRepository, - syncSchedulerClient, - destinationHandler); - - final JobHistoryHandler jobHistoryHandler = new JobHistoryHandler( - jobPersistence, - configs.getWorkerEnvironment(), - configs.getLogConfigs(), - connectionsHandler, - sourceHandler, - sourceDefinitionsHandler, - destinationHandler, - destinationDefinitionsHandler, - configs.getAirbyteVersion()); - - final WebBackendConnectionsHandler webBackendConnectionsHandler = new WebBackendConnectionsHandler( - connectionsHandler, - stateHandler, - sourceHandler, - destinationHandler, - jobHistoryHandler, - schedulerHandler, - operationsHandler, - eventRunner, - configRepository); - LOGGER.info("Starting server..."); return apiFactory.create( @@ -362,9 +304,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, attemptHandler, connectionsHandler, operationsHandler, - schedulerHandler, - stateHandler, - webBackendConnectionsHandler); + schedulerHandler); } @VisibleForTesting diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 6bced7164b34..cc9e7dbc0bd2 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -25,8 +25,6 @@ import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; -import io.airbyte.server.handlers.StateHandler; -import io.airbyte.server.handlers.WebBackendConnectionsHandler; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; @@ -57,9 +55,7 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien final AttemptHandler attemptHandler, final ConnectionsHandler connectionsHandler, final OperationsHandler operationsHandler, - final SchedulerHandler schedulerHandler, - final StateHandler stateHandler, - final WebBackendConnectionsHandler webBackendConnectionsHandler); + final SchedulerHandler schedulerHandler); class Api implements ServerFactory { @@ -83,9 +79,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final AttemptHandler attemptHandler, final ConnectionsHandler connectionsHandler, final OperationsHandler operationsHandler, - final SchedulerHandler schedulerHandler, - final StateHandler stateHandler, - final WebBackendConnectionsHandler webBackendConnectionsHandler) { + final SchedulerHandler schedulerHandler) { final Map mdc = MDC.getCopyOfContextMap(); // set static values for factory @@ -115,8 +109,6 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul connectionsHandler, operationsHandler, schedulerHandler, - stateHandler, - webBackendConnectionsHandler, mdc); // server configurations diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index 0d74d2a78c3b..716bfea0cb19 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -16,27 +16,19 @@ import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; -import io.airbyte.server.handlers.StateHandler; -import io.airbyte.server.handlers.WebBackendConnectionsHandler; public class ConnectionApiController implements ConnectionApi { private final ConnectionsHandler connectionsHandler; private final OperationsHandler operationsHandler; private final SchedulerHandler schedulerHandler; - private final StateHandler stateHandler; - private final WebBackendConnectionsHandler webBackendConnectionsHandler; public ConnectionApiController(final ConnectionsHandler connectionsHandler, final OperationsHandler operationsHandler, - final SchedulerHandler schedulerHandler, - final StateHandler stateHandler, - final WebBackendConnectionsHandler webBackendConnectionsHandler) { + final SchedulerHandler schedulerHandler) { this.connectionsHandler = connectionsHandler; this.operationsHandler = operationsHandler; this.schedulerHandler = schedulerHandler; - this.stateHandler = stateHandler; - this.webBackendConnectionsHandler = webBackendConnectionsHandler; } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java index 174a12ddc79d..7378d342b65f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java @@ -8,8 +8,6 @@ import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; -import io.airbyte.server.handlers.StateHandler; -import io.airbyte.server.handlers.WebBackendConnectionsHandler; import java.util.Map; import org.glassfish.hk2.api.Factory; import org.slf4j.MDC; @@ -19,21 +17,15 @@ public class ConnectionApiFactory implements Factory { private static ConnectionsHandler connectionsHandler; private static OperationsHandler operationsHandler; private static SchedulerHandler schedulerHandler; - private static StateHandler stateHandler; - private static WebBackendConnectionsHandler webBackendConnectionsHandler; private static Map mdc; public static void setValues(final ConnectionsHandler connectionsHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler, - final StateHandler stateHandler, - final WebBackendConnectionsHandler webBackendConnectionsHandler, final Map mdc) { ConnectionApiFactory.connectionsHandler = connectionsHandler; ConnectionApiFactory.operationsHandler = operationsHandler; ConnectionApiFactory.schedulerHandler = schedulerHandler; - ConnectionApiFactory.stateHandler = stateHandler; - ConnectionApiFactory.webBackendConnectionsHandler = webBackendConnectionsHandler; ConnectionApiFactory.mdc = mdc; } @@ -41,7 +33,7 @@ public static void setValues(final ConnectionsHandler connectionsHandler, public ConnectionApiController provide() { MDC.setContextMap(ConnectionApiFactory.mdc); - return new ConnectionApiController(connectionsHandler, operationsHandler, schedulerHandler, stateHandler, webBackendConnectionsHandler); + return new ConnectionApiController(connectionsHandler, operationsHandler, schedulerHandler); } @Override From 50fc3ae18fde762d316d1b194de6b4569adbffaa Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 16:01:48 -0700 Subject: [PATCH 14/34] Extract DB migrator --- .../server/ConfigurationApiFactory.java | 14 +------------ .../java/io/airbyte/server/ServerApp.java | 3 +++ .../java/io/airbyte/server/ServerFactory.java | 6 ++++++ .../airbyte/server/apis/ConfigurationApi.java | 8 +------ .../server/apis/DbMigrationApiController.java | 11 ++++++++-- .../apis/binders/DbMigrationBinder.java | 21 +++++++++++++++++++ .../apis/factories/DbMigrationApiFactory.java | 16 +++++++++++--- .../server/apis/ConfigurationApiTest.java | 8 +------ 8 files changed, 55 insertions(+), 32 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/DbMigrationBinder.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java index 4ed1af386612..297718c74fd9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java @@ -33,8 +33,6 @@ public class ConfigurationApiFactory implements Factory { private static SynchronousSchedulerClient synchronousSchedulerClient; private static StatePersistence statePersistence; private static Map mdc; - private static Database configsDatabase; - private static Database jobsDatabase; private static TrackingClient trackingClient; private static WorkerEnvironment workerEnvironment; private static LogConfigs logConfigs; @@ -42,8 +40,6 @@ public class ConfigurationApiFactory implements Factory { private static AirbyteVersion airbyteVersion; private static HttpClient httpClient; private static EventRunner eventRunner; - private static Flyway configsFlyway; - private static Flyway jobsFlyway; public static void setValues( final ConfigRepository configRepository, @@ -70,8 +66,6 @@ public static void setValues( ConfigurationApiFactory.secretsRepositoryWriter = secretsRepositoryWriter; ConfigurationApiFactory.synchronousSchedulerClient = synchronousSchedulerClient; ConfigurationApiFactory.mdc = mdc; - ConfigurationApiFactory.configsDatabase = configsDatabase; - ConfigurationApiFactory.jobsDatabase = jobsDatabase; ConfigurationApiFactory.trackingClient = trackingClient; ConfigurationApiFactory.workerEnvironment = workerEnvironment; ConfigurationApiFactory.logConfigs = logConfigs; @@ -79,8 +73,6 @@ public static void setValues( ConfigurationApiFactory.airbyteVersion = airbyteVersion; ConfigurationApiFactory.httpClient = httpClient; ConfigurationApiFactory.eventRunner = eventRunner; - ConfigurationApiFactory.configsFlyway = configsFlyway; - ConfigurationApiFactory.jobsFlyway = jobsFlyway; ConfigurationApiFactory.statePersistence = statePersistence; } @@ -94,8 +86,6 @@ public ConfigurationApi provide() { ConfigurationApiFactory.secretsRepositoryReader, ConfigurationApiFactory.secretsRepositoryWriter, ConfigurationApiFactory.synchronousSchedulerClient, - ConfigurationApiFactory.configsDatabase, - ConfigurationApiFactory.jobsDatabase, ConfigurationApiFactory.statePersistence, ConfigurationApiFactory.trackingClient, ConfigurationApiFactory.workerEnvironment, @@ -103,9 +93,7 @@ public ConfigurationApi provide() { ConfigurationApiFactory.airbyteVersion, ConfigurationApiFactory.workspaceRoot, ConfigurationApiFactory.httpClient, - ConfigurationApiFactory.eventRunner, - ConfigurationApiFactory.configsFlyway, - ConfigurationApiFactory.jobsFlyway); + ConfigurationApiFactory.eventRunner); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index f3a9ec0409dc..c78e7c02f1aa 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -340,6 +340,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, eventRunner, configRepository); + final DbMigrationHandler dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -361,6 +363,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, jobsFlyway, attemptHandler, connectionsHandler, + dbMigrationHandler, operationsHandler, schedulerHandler, stateHandler, diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 6bced7164b34..237cad97d299 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -21,8 +21,10 @@ import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; +import io.airbyte.server.apis.factories.DbMigrationApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; +import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.handlers.StateHandler; @@ -56,6 +58,7 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien final Flyway jobsFlyway, final AttemptHandler attemptHandler, final ConnectionsHandler connectionsHandler, + final DbMigrationHandler dbMigrationHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler, final StateHandler stateHandler, @@ -82,6 +85,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final Flyway jobsFlyway, final AttemptHandler attemptHandler, final ConnectionsHandler connectionsHandler, + final DbMigrationHandler dbMigrationHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler, final StateHandler stateHandler, @@ -119,6 +123,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul webBackendConnectionsHandler, mdc); + DbMigrationApiFactory.setValues(dbMigrationHandler, mdc); + // server configurations final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class); final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder()); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index f6b202b18d36..61bb7847270d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -109,7 +109,6 @@ import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.config.persistence.StatePersistence; -import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.server.errors.BadObjectSchemaKnownException; @@ -141,7 +140,6 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.NotImplementedException; -import org.flywaydb.core.Flyway; @javax.ws.rs.Path("/v1") @Slf4j @@ -172,8 +170,6 @@ public ConfigurationApi(final ConfigRepository configRepository, final SecretsRepositoryReader secretsRepositoryReader, final SecretsRepositoryWriter secretsRepositoryWriter, final SynchronousSchedulerClient synchronousSchedulerClient, - final Database configsDatabase, - final Database jobsDatabase, final StatePersistence statePersistence, final TrackingClient trackingClient, final WorkerEnvironment workerEnvironment, @@ -181,9 +177,7 @@ public ConfigurationApi(final ConfigRepository configRepository, final AirbyteVersion airbyteVersion, final Path workspaceRoot, final HttpClient httpClient, - final EventRunner eventRunner, - final Flyway configsFlyway, - final Flyway jobsFlyway) { + final EventRunner eventRunner) { this.workerEnvironment = workerEnvironment; this.logConfigs = logConfigs; this.workspaceRoot = workspaceRoot; diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java index bba8786c9d56..93cb85dd33a7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis; import io.airbyte.api.generated.DbMigrationApi; @@ -16,11 +20,14 @@ public DbMigrationApiController(final DbMigrationHandler dbMigrationHandler) { this.dbMigrationHandler = dbMigrationHandler; } - @Override public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { + @Override + public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { return ConfigurationApi.execute(() -> dbMigrationHandler.migrate(dbMigrationRequestBody)); } - @Override public DbMigrationReadList listMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { + @Override + public DbMigrationReadList listMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { return ConfigurationApi.execute(() -> dbMigrationHandler.list(dbMigrationRequestBody)); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DbMigrationBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DbMigrationBinder.java new file mode 100644 index 000000000000..78471be218d4 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DbMigrationBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.DbMigrationApiController; +import io.airbyte.server.apis.factories.DbMigrationApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class DbMigrationBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(DbMigrationApiFactory.class) + .to(DbMigrationApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java index d1b36e6ca5e7..ee4430f48e94 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java @@ -1,9 +1,14 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.factories; import io.airbyte.server.apis.DbMigrationApiController; import io.airbyte.server.handlers.DbMigrationHandler; import java.util.Map; import org.glassfish.hk2.api.Factory; +import org.slf4j.MDC; public class DbMigrationApiFactory implements Factory { @@ -15,11 +20,16 @@ public static void setValues(final DbMigrationHandler dbMigrationHandler, final DbMigrationApiFactory.mdc = mdc; } - @Override public DbMigrationApiController provide() { - return null; + @Override + public DbMigrationApiController provide() { + MDC.setContextMap(DbMigrationApiFactory.mdc); + + return new DbMigrationApiController(dbMigrationHandler); } - @Override public void dispose(final DbMigrationApiController instance) { + @Override + public void dispose(final DbMigrationApiController instance) { /* no op */ } + } diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java index 0bba057a25aa..34ec37f2cd4d 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java @@ -17,13 +17,11 @@ import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.config.persistence.StatePersistence; -import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; import java.nio.file.Path; -import org.flywaydb.core.Flyway; import org.junit.jupiter.api.Test; class ConfigurationApiTest { @@ -40,8 +38,6 @@ void testImportDefinitions() { mock(SecretsRepositoryReader.class), mock(SecretsRepositoryWriter.class), mock(SynchronousSchedulerClient.class), - mock(Database.class), - mock(Database.class), mock(StatePersistence.class), mock(TrackingClient.class), WorkerEnvironment.DOCKER, @@ -49,9 +45,7 @@ void testImportDefinitions() { new AirbyteVersion("0.1.0-alpha"), Path.of(""), mock(HttpClient.class), - mock(EventRunner.class), - mock(Flyway.class), - mock(Flyway.class)); + mock(EventRunner.class)); assertFalse(configurationApi.getHealthCheck().getAvailable()); } From 4e8051e0ee551c77c023bf02c8afe1fad0eb9881 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 25 Oct 2022 18:08:09 -0700 Subject: [PATCH 15/34] Add something that I forgot --- .../src/main/java/io/airbyte/server/ServerFactory.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 09276bdc1912..c659e8dc31bf 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -17,8 +17,10 @@ import io.airbyte.server.apis.AttemptApiController; import io.airbyte.server.apis.ConfigurationApi; import io.airbyte.server.apis.ConnectionApiController; +import io.airbyte.server.apis.DbMigrationApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; +import io.airbyte.server.apis.binders.DbMigrationBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; @@ -118,8 +120,10 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DbMigrationApiFactory.setValues(dbMigrationHandler, mdc); // server configurations - final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class); - final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder()); + final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class, + DbMigrationApiController.class); + final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder(), + new DbMigrationBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); From 7254ebb22e159ffac075e84b7438b1aceb301007 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 14:21:41 -0700 Subject: [PATCH 16/34] extract destination definition api --- .../java/io/airbyte/server/ServerApp.java | 5 + .../java/io/airbyte/server/ServerFactory.java | 14 ++- .../airbyte/server/apis/ConfigurationApi.java | 93 ++++++++++++++----- .../DestinationDefinitionApiController.java | 91 ++++++++++++++++++ .../DestinationDefinitionApiBinder.java | 15 +++ .../DestinationDefinitionApiFactory.java | 22 +++++ 6 files changed, 214 insertions(+), 26 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 3340a5411cbd..6eb2c4f7391e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -57,6 +57,7 @@ import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; @@ -295,6 +296,9 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final DbMigrationHandler dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); + final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, syncSchedulerClient, + destinationHandler); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -317,6 +321,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, attemptHandler, connectionsHandler, dbMigrationHandler, + destinationDefinitionsHandler, destinationHandler, operationsHandler, schedulerHandler); diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 76f81fedc2b8..e83b3684be9b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -19,16 +19,20 @@ import io.airbyte.server.apis.ConnectionApiController; import io.airbyte.server.apis.DbMigrationApiController; import io.airbyte.server.apis.DestinationApiController; +import io.airbyte.server.apis.DestinationDefinitionApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.binders.DbMigrationBinder; import io.airbyte.server.apis.binders.DestinationApiBinder; +import io.airbyte.server.apis.binders.DestinationDefinitionApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; +import io.airbyte.server.apis.factories.DestinationDefinitionApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; @@ -62,6 +66,7 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien final AttemptHandler attemptHandler, final ConnectionsHandler connectionsHandler, final DbMigrationHandler dbMigrationHandler, + final DestinationDefinitionsHandler destinationDefinitionsHandler, final DestinationHandler destinationApiHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler); @@ -88,6 +93,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final AttemptHandler attemptHandler, final ConnectionsHandler connectionsHandler, final DbMigrationHandler dbMigrationHandler, + final DestinationDefinitionsHandler destinationDefinitionsHandler, final DestinationHandler destinationApiHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler) { @@ -124,16 +130,18 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DbMigrationApiFactory.setValues(dbMigrationHandler, mdc); + DestinationDefinitionApiFactory.setValues(destinationDefinitionsHandler); + // server configurations final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class, - DbMigrationApiController.class, DestinationApiController.class); + DbMigrationApiController.class, DestinationApiController.class, DestinationDefinitionApiController.class); final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder(), - new DbMigrationBinder(), new DestinationApiBinder()); + new DbMigrationBinder(), new DestinationApiBinder(), new DestinationDefinitionApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); } - + } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index af525c0fa56a..e7e8125e3c91 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -510,87 +510,134 @@ public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody r // DESTINATION + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionReadList listDestinationDefinitions() { - return execute(destinationDefinitionsHandler::listDestinationDefinitions); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> destinationDefinitionsHandler.listDestinationDefinitionsForWorkspace(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionReadList listLatestDestinationDefinitions() { - return execute(destinationDefinitionsHandler::listLatestDestinationDefinitions); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { - return execute(() -> destinationDefinitionsHandler.getDestinationDefinition(destinationDefinitionIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionRead getDestinationDefinitionForWorkspace( final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return execute(() -> destinationDefinitionsHandler.getDestinationDefinitionForWorkspace(destinationDefinitionIdWithWorkspaceId)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ // TODO: Deprecate this route in favor of createCustomDestinationDefinition // since all connector definitions created through the API are custom @Override public DestinationDefinitionRead createDestinationDefinition(final DestinationDefinitionCreate destinationDefinitionCreate) { - return execute(() -> destinationDefinitionsHandler.createPrivateDestinationDefinition(destinationDefinitionCreate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { - return execute(() -> destinationDefinitionsHandler.createCustomDestinationDefinition(customDestinationDefinitionCreate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { - return execute(() -> destinationDefinitionsHandler.updateDestinationDefinition(destinationDefinitionUpdate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomDestinationDefinitionUpdate customDestinationDefinitionUpdate) { - return execute(() -> destinationDefinitionsHandler.updateCustomDestinationDefinition(customDestinationDefinitionUpdate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { - execute(() -> { - destinationDefinitionsHandler.deleteDestinationDefinition(destinationDefinitionIdRequestBody); - return null; - }); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - execute(() -> { - destinationDefinitionsHandler.deleteCustomDestinationDefinition(destinationDefinitionIdWithWorkspaceId); - return null; - }); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace( final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return execute(() -> destinationDefinitionsHandler.grantDestinationDefinitionToWorkspace(destinationDefinitionIdWithWorkspaceId)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of + * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + */ @Override public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - execute(() -> { - destinationDefinitionsHandler.revokeDestinationDefinitionFromWorkspace(destinationDefinitionIdWithWorkspaceId); - return null; - }); + throw new NotImplementedException(); } // DESTINATION SPECIFICATION diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java new file mode 100644 index 000000000000..1e8078ae79a9 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java @@ -0,0 +1,91 @@ +package io.airbyte.server.apis; + +import io.airbyte.api.generated.DestinationDefinitionApi; +import io.airbyte.api.model.generated.CustomDestinationDefinitionCreate; +import io.airbyte.api.model.generated.CustomDestinationDefinitionUpdate; +import io.airbyte.api.model.generated.DestinationDefinitionCreate; +import io.airbyte.api.model.generated.DestinationDefinitionIdRequestBody; +import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.DestinationDefinitionRead; +import io.airbyte.api.model.generated.DestinationDefinitionReadList; +import io.airbyte.api.model.generated.DestinationDefinitionUpdate; +import io.airbyte.api.model.generated.PrivateDestinationDefinitionRead; +import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList; +import io.airbyte.api.model.generated.WorkspaceIdRequestBody; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class DestinationDefinitionApiController implements DestinationDefinitionApi { + + private final DestinationDefinitionsHandler destinationDefinitionsHandler; + + @Override public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.createCustomDestinationDefinition(customDestinationDefinitionCreate)); + } + + // TODO: Deprecate this route in favor of createCustomDestinationDefinition + // since all connector definitions created through the API are custom + @Override public DestinationDefinitionRead createDestinationDefinition(final DestinationDefinitionCreate destinationDefinitionCreate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.createPrivateDestinationDefinition(destinationDefinitionCreate)); + } + + @Override public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + ConfigurationApi.execute(() -> { + destinationDefinitionsHandler.deleteCustomDestinationDefinition(destinationDefinitionIdWithWorkspaceId); + return null; + }); + } + + @Override public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { + ConfigurationApi.execute(() -> { + destinationDefinitionsHandler.deleteDestinationDefinition(destinationDefinitionIdRequestBody); + return null; + }); + } + + @Override public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.getDestinationDefinition(destinationDefinitionIdRequestBody)); + } + + @Override + public DestinationDefinitionRead getDestinationDefinitionForWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.getDestinationDefinitionForWorkspace(destinationDefinitionIdWithWorkspaceId)); + } + + @Override + public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.grantDestinationDefinitionToWorkspace(destinationDefinitionIdWithWorkspaceId)); + } + + @Override public DestinationDefinitionReadList listDestinationDefinitions() { + return ConfigurationApi.execute(destinationDefinitionsHandler::listDestinationDefinitions); + } + + @Override public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.listDestinationDefinitionsForWorkspace(workspaceIdRequestBody)); + } + + @Override public DestinationDefinitionReadList listLatestDestinationDefinitions() { + return ConfigurationApi.execute(destinationDefinitionsHandler::listLatestDestinationDefinitions); + } + + @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); + } + + @Override public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + ConfigurationApi.execute(() -> { + destinationDefinitionsHandler.revokeDestinationDefinitionFromWorkspace(destinationDefinitionIdWithWorkspaceId); + return null; + }); + } + + @Override public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomDestinationDefinitionUpdate customDestinationDefinitionUpdate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.updateCustomDestinationDefinition(customDestinationDefinitionUpdate)); + } + + @Override public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.updateDestinationDefinition(destinationDefinitionUpdate)); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java new file mode 100644 index 000000000000..a0160037b8fa --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java @@ -0,0 +1,15 @@ +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.DestinationDefinitionApiController; +import io.airbyte.server.apis.factories.DestinationDefinitionApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class DestinationDefinitionApiBinder extends AbstractBinder { + + @Override protected void configure() { + bindFactory(DestinationDefinitionApiFactory.class) + .to(DestinationDefinitionApiController.class) + .in(RequestScoped.class); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java new file mode 100644 index 000000000000..db497afd8888 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java @@ -0,0 +1,22 @@ +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.DestinationDefinitionApiController; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import org.glassfish.hk2.api.Factory; + +public class DestinationDefinitionApiFactory implements Factory { + + private static DestinationDefinitionsHandler destinationDefinitionsHandler; + + public static void setValues(final DestinationDefinitionsHandler destinationDefinitionsHandler) { + DestinationDefinitionApiFactory.destinationDefinitionsHandler = destinationDefinitionsHandler; + } + + @Override public DestinationDefinitionApiController provide() { + return new DestinationDefinitionApiController(destinationDefinitionsHandler); + } + + @Override public void dispose(final DestinationDefinitionApiController instance) { + /* no op */ + } +} From 4d748afcff87e986f877cecca58bbc67f38c1c67 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 14:23:18 -0700 Subject: [PATCH 17/34] restore destination factory initialization --- .../src/main/java/io/airbyte/server/ServerFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 76f81fedc2b8..8467e633edd7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -26,6 +26,7 @@ import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; +import io.airbyte.server.apis.factories.DestinationApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; @@ -124,6 +125,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DbMigrationApiFactory.setValues(dbMigrationHandler, mdc); + DestinationApiFactory.setValues(destinationApiHandler, schedulerHandler, mdc); + // server configurations final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class, DbMigrationApiController.class, DestinationApiController.class); From e6a56a2fb571f208a49fe7bd548ff73f77c687ac Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 14:35:35 -0700 Subject: [PATCH 18/34] extract destination definition specification api --- .../java/io/airbyte/server/ServerFactory.java | 10 +++++++-- .../airbyte/server/apis/ConfigurationApi.java | 7 ++++-- ...nDefinitionSpecificationApiController.java | 18 +++++++++++++++ ...ationDefinitionSpecificationApiBinder.java | 15 +++++++++++++ ...tionDefinitionSpecificationApiFactory.java | 22 +++++++++++++++++++ 5 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index e83b3684be9b..a21a259cc2d7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -20,15 +20,18 @@ import io.airbyte.server.apis.DbMigrationApiController; import io.airbyte.server.apis.DestinationApiController; import io.airbyte.server.apis.DestinationDefinitionApiController; +import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.binders.DbMigrationBinder; import io.airbyte.server.apis.binders.DestinationApiBinder; import io.airbyte.server.apis.binders.DestinationDefinitionApiBinder; +import io.airbyte.server.apis.binders.DestinationDefinitionSpecificationApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; import io.airbyte.server.apis.factories.DestinationDefinitionApiFactory; +import io.airbyte.server.apis.factories.DestinationDefinitionSpecificationApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; @@ -132,11 +135,14 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DestinationDefinitionApiFactory.setValues(destinationDefinitionsHandler); + DestinationDefinitionSpecificationApiFactory.setValues(schedulerHandler); + // server configurations final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class, - DbMigrationApiController.class, DestinationApiController.class, DestinationDefinitionApiController.class); + DbMigrationApiController.class, DestinationApiController.class, DestinationDefinitionApiController.class, + DestinationDefinitionSpecificationApiController.class); final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder(), - new DbMigrationBinder(), new DestinationApiBinder(), new DestinationDefinitionApiBinder()); + new DbMigrationBinder(), new DestinationApiBinder(), new DestinationDefinitionApiBinder(), new DestinationDefinitionSpecificationApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index e7e8125e3c91..9a2afaa13784 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -641,11 +641,14 @@ public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinition } // DESTINATION SPECIFICATION - + /** + * This implementation has been moved to {@link DestinationDefinitionSpecificationApiController}. Since the path of + * {@link DestinationDefinitionSpecificationApiController} is more granular, it will override this implementation + */ @Override public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification( final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); + throw new NotImplementedException(); } // DESTINATION IMPLEMENTATION diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java new file mode 100644 index 000000000000..2a70e304648b --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java @@ -0,0 +1,18 @@ +package io.airbyte.server.apis; + +import io.airbyte.api.generated.DestinationDefinitionSpecificationApi; +import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; +import io.airbyte.server.handlers.SchedulerHandler; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class DestinationDefinitionSpecificationApiController implements DestinationDefinitionSpecificationApi { + + private final SchedulerHandler schedulerHandler; + + @Override + public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + return ConfigurationApi.execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java new file mode 100644 index 000000000000..65467d45acdf --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java @@ -0,0 +1,15 @@ +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; +import io.airbyte.server.apis.factories.DestinationDefinitionSpecificationApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class DestinationDefinitionSpecificationApiBinder extends AbstractBinder { + + @Override protected void configure() { + bindFactory(DestinationDefinitionSpecificationApiFactory.class) + .to(DestinationDefinitionSpecificationApiController.class) + .in(RequestScoped.class); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java new file mode 100644 index 000000000000..d69de1650a55 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java @@ -0,0 +1,22 @@ +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; +import io.airbyte.server.handlers.SchedulerHandler; +import org.glassfish.hk2.api.Factory; + +public class DestinationDefinitionSpecificationApiFactory implements Factory { + + private static SchedulerHandler schedulerHandler; + + public static void setValues(final SchedulerHandler schedulerHandler) { + DestinationDefinitionSpecificationApiFactory.schedulerHandler = schedulerHandler; + } + + @Override public DestinationDefinitionSpecificationApiController provide() { + return new DestinationDefinitionSpecificationApiController(DestinationDefinitionSpecificationApiFactory.schedulerHandler); + } + + @Override public void dispose(final DestinationDefinitionSpecificationApiController instance) { + + } +} From 709cdc430e76567fdfefbdfe445893835cca86ac Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 15:34:53 -0700 Subject: [PATCH 19/34] format --- .../src/main/java/io/airbyte/server/ServerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 8467e633edd7..a366a5afeff4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -136,7 +136,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul // construct server return new ServerApp(airbyteVersion, componentClasses, components); } - + } } From 82360dcd8d326f6f67bce1798de322b4b643be81 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 15:37:04 -0700 Subject: [PATCH 20/34] format --- .../airbyte/server/apis/ConfigurationApi.java | 70 +++++++++++-------- .../DestinationDefinitionApiController.java | 44 ++++++++---- .../DestinationDefinitionApiBinder.java | 8 ++- .../DestinationDefinitionApiFactory.java | 11 ++- 4 files changed, 89 insertions(+), 44 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index e7e8125e3c91..7ec2e9e45cd4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -511,8 +511,9 @@ public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody r // DESTINATION /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionReadList listDestinationDefinitions() { @@ -520,8 +521,9 @@ public DestinationDefinitionReadList listDestinationDefinitions() { } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { @@ -529,8 +531,9 @@ public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(fina } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionReadList listLatestDestinationDefinitions() { @@ -538,8 +541,9 @@ public DestinationDefinitionReadList listLatestDestinationDefinitions() { } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { @@ -547,8 +551,9 @@ public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(fi } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { @@ -556,8 +561,9 @@ public DestinationDefinitionRead getDestinationDefinition(final DestinationDefin } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionRead getDestinationDefinitionForWorkspace( @@ -566,8 +572,9 @@ public DestinationDefinitionRead getDestinationDefinitionForWorkspace( } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ // TODO: Deprecate this route in favor of createCustomDestinationDefinition // since all connector definitions created through the API are custom @@ -577,8 +584,9 @@ public DestinationDefinitionRead createDestinationDefinition(final DestinationDe } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { @@ -586,8 +594,9 @@ public DestinationDefinitionRead createCustomDestinationDefinition(final CustomD } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { @@ -595,8 +604,9 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomDestinationDefinitionUpdate customDestinationDefinitionUpdate) { @@ -604,8 +614,9 @@ public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomD } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { @@ -613,8 +624,9 @@ public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { @@ -622,8 +634,9 @@ public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithW } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace( @@ -632,8 +645,9 @@ public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace( } /** - * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path of - * {@link DestinationDefinitionApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation */ @Override public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java index 1e8078ae79a9..ba7495bc37e2 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis; import io.airbyte.api.generated.DestinationDefinitionApi; @@ -20,31 +24,36 @@ public class DestinationDefinitionApiController implements DestinationDefinition private final DestinationDefinitionsHandler destinationDefinitionsHandler; - @Override public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { + @Override + public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { return ConfigurationApi.execute(() -> destinationDefinitionsHandler.createCustomDestinationDefinition(customDestinationDefinitionCreate)); } // TODO: Deprecate this route in favor of createCustomDestinationDefinition // since all connector definitions created through the API are custom - @Override public DestinationDefinitionRead createDestinationDefinition(final DestinationDefinitionCreate destinationDefinitionCreate) { + @Override + public DestinationDefinitionRead createDestinationDefinition(final DestinationDefinitionCreate destinationDefinitionCreate) { return ConfigurationApi.execute(() -> destinationDefinitionsHandler.createPrivateDestinationDefinition(destinationDefinitionCreate)); } - @Override public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + @Override + public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { ConfigurationApi.execute(() -> { destinationDefinitionsHandler.deleteCustomDestinationDefinition(destinationDefinitionIdWithWorkspaceId); return null; }); } - @Override public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { + @Override + public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { ConfigurationApi.execute(() -> { destinationDefinitionsHandler.deleteDestinationDefinition(destinationDefinitionIdRequestBody); return null; }); } - @Override public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { + @Override + public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { return ConfigurationApi.execute(() -> destinationDefinitionsHandler.getDestinationDefinition(destinationDefinitionIdRequestBody)); } @@ -55,37 +64,46 @@ public DestinationDefinitionRead getDestinationDefinitionForWorkspace(final Dest @Override public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return ConfigurationApi.execute(() -> destinationDefinitionsHandler.grantDestinationDefinitionToWorkspace(destinationDefinitionIdWithWorkspaceId)); + return ConfigurationApi + .execute(() -> destinationDefinitionsHandler.grantDestinationDefinitionToWorkspace(destinationDefinitionIdWithWorkspaceId)); } - @Override public DestinationDefinitionReadList listDestinationDefinitions() { + @Override + public DestinationDefinitionReadList listDestinationDefinitions() { return ConfigurationApi.execute(destinationDefinitionsHandler::listDestinationDefinitions); } - @Override public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + @Override + public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ConfigurationApi.execute(() -> destinationDefinitionsHandler.listDestinationDefinitionsForWorkspace(workspaceIdRequestBody)); } - @Override public DestinationDefinitionReadList listLatestDestinationDefinitions() { + @Override + public DestinationDefinitionReadList listLatestDestinationDefinitions() { return ConfigurationApi.execute(destinationDefinitionsHandler::listLatestDestinationDefinitions); } - @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { + @Override + public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { return ConfigurationApi.execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); } - @Override public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + @Override + public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { ConfigurationApi.execute(() -> { destinationDefinitionsHandler.revokeDestinationDefinitionFromWorkspace(destinationDefinitionIdWithWorkspaceId); return null; }); } - @Override public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomDestinationDefinitionUpdate customDestinationDefinitionUpdate) { + @Override + public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomDestinationDefinitionUpdate customDestinationDefinitionUpdate) { return ConfigurationApi.execute(() -> destinationDefinitionsHandler.updateCustomDestinationDefinition(customDestinationDefinitionUpdate)); } - @Override public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { + @Override + public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { return ConfigurationApi.execute(() -> destinationDefinitionsHandler.updateDestinationDefinition(destinationDefinitionUpdate)); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java index a0160037b8fa..1d9279119796 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.binders; import io.airbyte.server.apis.DestinationDefinitionApiController; @@ -7,9 +11,11 @@ public class DestinationDefinitionApiBinder extends AbstractBinder { - @Override protected void configure() { + @Override + protected void configure() { bindFactory(DestinationDefinitionApiFactory.class) .to(DestinationDefinitionApiController.class) .in(RequestScoped.class); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java index db497afd8888..f76f631145ce 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.factories; import io.airbyte.server.apis.DestinationDefinitionApiController; @@ -12,11 +16,14 @@ public static void setValues(final DestinationDefinitionsHandler destinationDefi DestinationDefinitionApiFactory.destinationDefinitionsHandler = destinationDefinitionsHandler; } - @Override public DestinationDefinitionApiController provide() { + @Override + public DestinationDefinitionApiController provide() { return new DestinationDefinitionApiController(destinationDefinitionsHandler); } - @Override public void dispose(final DestinationDefinitionApiController instance) { + @Override + public void dispose(final DestinationDefinitionApiController instance) { /* no op */ } + } From 6ecdb17f9026f916885a4baf45b266fb042ab42f Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 15:39:09 -0700 Subject: [PATCH 21/34] format --- .../main/java/io/airbyte/server/ServerFactory.java | 3 ++- .../java/io/airbyte/server/apis/ConfigurationApi.java | 5 +++-- ...stinationDefinitionSpecificationApiController.java | 5 +++++ .../DestinationDefinitionSpecificationApiBinder.java | 8 +++++++- .../DestinationDefinitionSpecificationApiFactory.java | 11 +++++++++-- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index ff3ea877dfad..8a6e6246bc5f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -145,7 +145,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DbMigrationApiController.class, DestinationApiController.class, DestinationDefinitionApiController.class, DestinationDefinitionSpecificationApiController.class); final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder(), - new DbMigrationBinder(), new DestinationApiBinder(), new DestinationDefinitionApiBinder(), new DestinationDefinitionSpecificationApiBinder()); + new DbMigrationBinder(), new DestinationApiBinder(), new DestinationDefinitionApiBinder(), + new DestinationDefinitionSpecificationApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index fee730e27cb2..f6172b53dea1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -656,8 +656,9 @@ public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinition // DESTINATION SPECIFICATION /** - * This implementation has been moved to {@link DestinationDefinitionSpecificationApiController}. Since the path of - * {@link DestinationDefinitionSpecificationApiController} is more granular, it will override this implementation + * This implementation has been moved to {@link DestinationDefinitionSpecificationApiController}. + * Since the path of {@link DestinationDefinitionSpecificationApiController} is more granular, it + * will override this implementation */ @Override public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification( diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java index 2a70e304648b..5344637f0e08 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis; import io.airbyte.api.generated.DestinationDefinitionSpecificationApi; @@ -15,4 +19,5 @@ public class DestinationDefinitionSpecificationApiController implements Destinat public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { return ConfigurationApi.execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java index 65467d45acdf..3d166bd139af 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.binders; import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; @@ -7,9 +11,11 @@ public class DestinationDefinitionSpecificationApiBinder extends AbstractBinder { - @Override protected void configure() { + @Override + protected void configure() { bindFactory(DestinationDefinitionSpecificationApiFactory.class) .to(DestinationDefinitionSpecificationApiController.class) .in(RequestScoped.class); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java index d69de1650a55..d7dcbfac229d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.factories; import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; @@ -12,11 +16,14 @@ public static void setValues(final SchedulerHandler schedulerHandler) { DestinationDefinitionSpecificationApiFactory.schedulerHandler = schedulerHandler; } - @Override public DestinationDefinitionSpecificationApiController provide() { + @Override + public DestinationDefinitionSpecificationApiController provide() { return new DestinationDefinitionSpecificationApiController(DestinationDefinitionSpecificationApiFactory.schedulerHandler); } - @Override public void dispose(final DestinationDefinitionSpecificationApiController instance) { + @Override + public void dispose(final DestinationDefinitionSpecificationApiController instance) { } + } From 187c9eabfe8422ea0a9f99f98fe9e1d220eea733 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 15:56:09 -0700 Subject: [PATCH 22/34] extract health check api --- .../java/io/airbyte/server/ServerApp.java | 4 +++ .../java/io/airbyte/server/ServerFactory.java | 34 +++++++++++++++---- .../airbyte/server/apis/ConfigurationApi.java | 6 +++- .../server/apis/HealthApiController.java | 24 +++++++++++++ .../server/apis/binders/HealthApiBinder.java | 21 ++++++++++++ .../apis/factories/HealthApiFactory.java | 29 ++++++++++++++++ 6 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/HealthApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/HealthApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 6eb2c4f7391e..8ec8ac60b824 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -59,6 +59,7 @@ import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.HealthCheckHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient; @@ -299,6 +300,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, syncSchedulerClient, destinationHandler); + final HealthCheckHandler healthCheckHandler = new HealthCheckHandler(configRepository); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -323,6 +326,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, dbMigrationHandler, destinationDefinitionsHandler, destinationHandler, + healthCheckHandler, operationsHandler, schedulerHandler); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 8a6e6246bc5f..42a18c32ddc6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -21,23 +21,27 @@ import io.airbyte.server.apis.DestinationApiController; import io.airbyte.server.apis.DestinationDefinitionApiController; import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; +import io.airbyte.server.apis.HealthApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.binders.DbMigrationBinder; import io.airbyte.server.apis.binders.DestinationApiBinder; import io.airbyte.server.apis.binders.DestinationDefinitionApiBinder; import io.airbyte.server.apis.binders.DestinationDefinitionSpecificationApiBinder; +import io.airbyte.server.apis.binders.HealthApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; import io.airbyte.server.apis.factories.DestinationApiFactory; import io.airbyte.server.apis.factories.DestinationDefinitionApiFactory; import io.airbyte.server.apis.factories.DestinationDefinitionSpecificationApiFactory; +import io.airbyte.server.apis.factories.HealthApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.HealthCheckHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.scheduler.EventRunner; @@ -72,6 +76,7 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien final DbMigrationHandler dbMigrationHandler, final DestinationDefinitionsHandler destinationDefinitionsHandler, final DestinationHandler destinationApiHandler, + final HealthCheckHandler healthCheckHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler); @@ -99,6 +104,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final DbMigrationHandler dbMigrationHandler, final DestinationDefinitionsHandler destinationDefinitionsHandler, final DestinationHandler destinationApiHandler, + final HealthCheckHandler healthCheckHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler) { final Map mdc = MDC.getCopyOfContextMap(); @@ -140,13 +146,29 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DestinationDefinitionSpecificationApiFactory.setValues(schedulerHandler); + HealthApiFactory.setValues(healthCheckHandler); + // server configurations - final Set> componentClasses = Set.of(ConfigurationApi.class, AttemptApiController.class, ConnectionApiController.class, - DbMigrationApiController.class, DestinationApiController.class, DestinationDefinitionApiController.class, - DestinationDefinitionSpecificationApiController.class); - final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder(), new AttemptApiBinder(), new ConnectionApiBinder(), - new DbMigrationBinder(), new DestinationApiBinder(), new DestinationDefinitionApiBinder(), - new DestinationDefinitionSpecificationApiBinder()); + final Set> componentClasses = Set.of( + ConfigurationApi.class, + AttemptApiController.class, + ConnectionApiController.class, + DbMigrationApiController.class, + DestinationApiController.class, + DestinationDefinitionApiController.class, + DestinationDefinitionSpecificationApiController.class, + HealthApiController.class); + + final Set components = Set.of( + new CorsFilter(), + new ConfigurationApiBinder(), + new AttemptApiBinder(), + new ConnectionApiBinder(), + new DbMigrationBinder(), + new DestinationApiBinder(), + new DestinationDefinitionApiBinder(), + new DestinationDefinitionSpecificationApiBinder(), + new HealthApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index f6172b53dea1..08d32415592a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -936,9 +936,13 @@ public File getOpenApiSpec() { } // HEALTH + /** + * This implementation has been moved to {@link HealthApiController}. Since the path of + * {@link HealthApiController} is more granular, it will override this implementation + */ @Override public HealthCheckRead getHealthCheck() { - return healthCheckHandler.health(); + throw new NotImplementedException(); } // WEB BACKEND diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java new file mode 100644 index 000000000000..75c10ca4852b --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.HealthApi; +import io.airbyte.api.model.generated.HealthCheckRead; +import io.airbyte.server.handlers.HealthCheckHandler; +import javax.ws.rs.Path; +import lombok.AllArgsConstructor; + +@Path("/v1/health") +@AllArgsConstructor +public class HealthApiController implements HealthApi { + + private final HealthCheckHandler healthCheckHandler; + + @Override + public HealthCheckRead getHealthCheck() { + return healthCheckHandler.health(); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/HealthApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/HealthApiBinder.java new file mode 100644 index 000000000000..bfe6161529f8 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/HealthApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.HealthApiController; +import io.airbyte.server.apis.factories.HealthApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class HealthApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(HealthApiFactory.class) + .to(HealthApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/HealthApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/HealthApiFactory.java new file mode 100644 index 000000000000..b13c17d5ebb2 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/HealthApiFactory.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.HealthApiController; +import io.airbyte.server.handlers.HealthCheckHandler; +import org.glassfish.hk2.api.Factory; + +public class HealthApiFactory implements Factory { + + private static HealthCheckHandler healthCheckHandler; + + public static void setValues(final HealthCheckHandler healthCheckHandler) { + HealthApiFactory.healthCheckHandler = healthCheckHandler; + } + + @Override + public HealthApiController provide() { + return new HealthApiController(healthCheckHandler); + } + + @Override + public void dispose(final HealthApiController instance) { + /* no op */ + } + +} From f6a5e9b8be862d8332b5dea1c63ac818a64fd35e Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 26 Oct 2022 16:32:07 -0700 Subject: [PATCH 23/34] extract jobs api --- .../java/io/airbyte/server/ServerApp.java | 24 ++++++++ .../java/io/airbyte/server/ServerFactory.java | 6 ++ .../airbyte/server/apis/ConfigurationApi.java | 34 +++++++++-- .../server/apis/JobsApiController.java | 57 +++++++++++++++++++ .../server/apis/binders/JobsApiBinder.java | 21 +++++++ .../server/apis/factories/JobsApiFactory.java | 32 +++++++++++ 6 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/JobsApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/JobsApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 8ec8ac60b824..082ee1221f5c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -60,8 +60,11 @@ import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.HealthCheckHandler; +import io.airbyte.server.handlers.JobHistoryHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; +import io.airbyte.server.handlers.SourceDefinitionsHandler; +import io.airbyte.server.handlers.SourceHandler; import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.TemporalEventRunner; @@ -302,6 +305,26 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final HealthCheckHandler healthCheckHandler = new HealthCheckHandler(configRepository); + final SourceHandler sourceHandler = new SourceHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + schemaValidator, + connectionsHandler); + + final SourceDefinitionsHandler sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, syncSchedulerClient, sourceHandler); + + final JobHistoryHandler jobHistoryHandler = new JobHistoryHandler( + jobPersistence, + configs.getWorkerEnvironment(), + configs.getLogConfigs(), + connectionsHandler, + sourceHandler, + sourceDefinitionsHandler, + destinationHandler, + destinationDefinitionsHandler, + configs.getAirbyteVersion()); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -327,6 +350,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, destinationDefinitionsHandler, destinationHandler, healthCheckHandler, + jobHistoryHandler, operationsHandler, schedulerHandler); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 42a18c32ddc6..5ecc963db3c6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -36,12 +36,14 @@ import io.airbyte.server.apis.factories.DestinationDefinitionApiFactory; import io.airbyte.server.apis.factories.DestinationDefinitionSpecificationApiFactory; import io.airbyte.server.apis.factories.HealthApiFactory; +import io.airbyte.server.apis.factories.JobsApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.HealthCheckHandler; +import io.airbyte.server.handlers.JobHistoryHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.scheduler.EventRunner; @@ -77,6 +79,7 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien final DestinationDefinitionsHandler destinationDefinitionsHandler, final DestinationHandler destinationApiHandler, final HealthCheckHandler healthCheckHandler, + final JobHistoryHandler jobHistoryHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler); @@ -105,6 +108,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final DestinationDefinitionsHandler destinationDefinitionsHandler, final DestinationHandler destinationApiHandler, final HealthCheckHandler healthCheckHandler, + final JobHistoryHandler jobHistoryHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler) { final Map mdc = MDC.getCopyOfContextMap(); @@ -148,6 +152,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul HealthApiFactory.setValues(healthCheckHandler); + JobsApiFactory.setValues(jobHistoryHandler, schedulerHandler); + // server configurations final Set> componentClasses = Set.of( ConfigurationApi.class, diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 08d32415592a..c5261f13c8c7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -893,33 +893,57 @@ public SourceDiscoverSchemaRead executeSourceDiscoverSchema(final SourceCoreConf return execute(() -> schedulerHandler.discoverSchemaForSourceFromSourceCreate(sourceCreate)); } + /** + * This implementation has been moved to {@link JobsApiController}. Since the path of + * {@link JobsApiController} is more granular, it will override this implementation + */ @Override public JobInfoRead cancelJob(final JobIdRequestBody jobIdRequestBody) { - return execute(() -> schedulerHandler.cancelJob(jobIdRequestBody)); + throw new NotImplementedException(); } // JOB HISTORY + /** + * This implementation has been moved to {@link JobsApiController}. Since the path of + * {@link JobsApiController} is more granular, it will override this implementation + */ @Override public JobReadList listJobsFor(final JobListRequestBody jobListRequestBody) { - return execute(() -> jobHistoryHandler.listJobsFor(jobListRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link JobsApiController}. Since the path of + * {@link JobsApiController} is more granular, it will override this implementation + */ @Override public JobInfoRead getJobInfo(final JobIdRequestBody jobIdRequestBody) { - return execute(() -> jobHistoryHandler.getJobInfo(jobIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link JobsApiController}. Since the path of + * {@link JobsApiController} is more granular, it will override this implementation + */ @Override public JobInfoLightRead getJobInfoLight(final JobIdRequestBody jobIdRequestBody) { - return execute(() -> jobHistoryHandler.getJobInfoLight(jobIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link JobsApiController}. Since the path of + * {@link JobsApiController} is more granular, it will override this implementation + */ @Override public JobDebugInfoRead getJobDebugInfo(final JobIdRequestBody jobIdRequestBody) { - return execute(() -> jobHistoryHandler.getJobDebugInfo(jobIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link JobsApiController}. Since the path of + * {@link JobsApiController} is more granular, it will override this implementation + */ @Override public AttemptNormalizationStatusReadList getAttemptNormalizationStatusesForJob(final JobIdRequestBody jobIdRequestBody) { return execute(() -> jobHistoryHandler.getAttemptNormalizationStatuses(jobIdRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java new file mode 100644 index 000000000000..0c1eb3a276a1 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/JobsApiController.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.JobsApi; +import io.airbyte.api.model.generated.AttemptNormalizationStatusReadList; +import io.airbyte.api.model.generated.JobDebugInfoRead; +import io.airbyte.api.model.generated.JobIdRequestBody; +import io.airbyte.api.model.generated.JobInfoLightRead; +import io.airbyte.api.model.generated.JobInfoRead; +import io.airbyte.api.model.generated.JobListRequestBody; +import io.airbyte.api.model.generated.JobReadList; +import io.airbyte.server.handlers.JobHistoryHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import javax.ws.rs.Path; +import lombok.AllArgsConstructor; + +@Path("/v1/jobs") +@AllArgsConstructor +public class JobsApiController implements JobsApi { + + private final JobHistoryHandler jobHistoryHandler; + private final SchedulerHandler schedulerHandler; + + @Override + public JobInfoRead cancelJob(final JobIdRequestBody jobIdRequestBody) { + return ConfigurationApi.execute(() -> schedulerHandler.cancelJob(jobIdRequestBody)); + } + + @Override + public AttemptNormalizationStatusReadList getAttemptNormalizationStatusesForJob(final JobIdRequestBody jobIdRequestBody) { + return ConfigurationApi.execute(() -> jobHistoryHandler.getAttemptNormalizationStatuses(jobIdRequestBody)); + } + + @Override + public JobDebugInfoRead getJobDebugInfo(final JobIdRequestBody jobIdRequestBody) { + return ConfigurationApi.execute(() -> jobHistoryHandler.getJobDebugInfo(jobIdRequestBody)); + } + + @Override + public JobInfoRead getJobInfo(final JobIdRequestBody jobIdRequestBody) { + return ConfigurationApi.execute(() -> jobHistoryHandler.getJobInfo(jobIdRequestBody)); + } + + @Override + public JobInfoLightRead getJobInfoLight(final JobIdRequestBody jobIdRequestBody) { + return ConfigurationApi.execute(() -> jobHistoryHandler.getJobInfoLight(jobIdRequestBody)); + } + + @Override + public JobReadList listJobsFor(final JobListRequestBody jobListRequestBody) { + return ConfigurationApi.execute(() -> jobHistoryHandler.listJobsFor(jobListRequestBody)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/JobsApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/JobsApiBinder.java new file mode 100644 index 000000000000..8f96f02357c0 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/JobsApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.JobsApiController; +import io.airbyte.server.apis.factories.JobsApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class JobsApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(JobsApiFactory.class) + .to(JobsApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/JobsApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/JobsApiFactory.java new file mode 100644 index 000000000000..618a15dc1bf0 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/JobsApiFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.JobsApiController; +import io.airbyte.server.handlers.JobHistoryHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import org.glassfish.hk2.api.Factory; + +public class JobsApiFactory implements Factory { + + private static JobHistoryHandler jobHistoryHandler; + private static SchedulerHandler schedulerHandler; + + public static void setValues(final JobHistoryHandler jobHistoryHandler, final SchedulerHandler schedulerHandler) { + JobsApiFactory.jobHistoryHandler = jobHistoryHandler; + JobsApiFactory.schedulerHandler = schedulerHandler; + } + + @Override + public JobsApiController provide() { + return new JobsApiController(jobHistoryHandler, schedulerHandler); + } + + @Override + public void dispose(final JobsApiController instance) { + /* no op */ + } + +} From 8023bc96a0c0e29d0a31996a1378da7d09915de2 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Thu, 27 Oct 2022 10:11:39 -0400 Subject: [PATCH 24/34] fix test --- .../airbyte/server/apis/ConfigurationApi.java | 4 -- .../server/apis/ConfigurationApiTest.java | 53 ------------------- .../server/apis/HealthCheckApiTest.java | 30 +++++++++++ 3 files changed, 30 insertions(+), 57 deletions(-) delete mode 100644 airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java create mode 100644 airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 08d32415592a..6bc240c92a41 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -116,7 +116,6 @@ import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; -import io.airbyte.server.handlers.HealthCheckHandler; import io.airbyte.server.handlers.JobHistoryHandler; import io.airbyte.server.handlers.LogsHandler; import io.airbyte.server.handlers.OAuthHandler; @@ -157,7 +156,6 @@ public class ConfigurationApi implements io.airbyte.api.generated.V1Api { private final JobHistoryHandler jobHistoryHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; private final WebBackendGeographiesHandler webBackendGeographiesHandler; - private final HealthCheckHandler healthCheckHandler; private final LogsHandler logsHandler; private final OpenApiConfigHandler openApiConfigHandler; private final OAuthHandler oAuthHandler; @@ -169,7 +167,6 @@ public ConfigurationApi(final ConfigRepository configRepository, final JobPersistence jobPersistence, final SecretsRepositoryReader secretsRepositoryReader, final SecretsRepositoryWriter secretsRepositoryWriter, - final SynchronousSchedulerClient synchronousSchedulerClient, final StatePersistence statePersistence, final TrackingClient trackingClient, @@ -238,7 +235,6 @@ public ConfigurationApi(final ConfigRepository configRepository, eventRunner, configRepository); webBackendGeographiesHandler = new WebBackendGeographiesHandler(); - healthCheckHandler = new HealthCheckHandler(configRepository); logsHandler = new LogsHandler(); openApiConfigHandler = new OpenApiConfigHandler(); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java deleted file mode 100644 index 34ec37f2cd4d..000000000000 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.apis; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.airbyte.analytics.TrackingClient; -import io.airbyte.commons.version.AirbyteVersion; -import io.airbyte.config.Configs; -import io.airbyte.config.Configs.WorkerEnvironment; -import io.airbyte.config.helpers.LogConfigs; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.SecretsRepositoryReader; -import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.config.persistence.StatePersistence; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.server.scheduler.EventRunner; -import io.airbyte.server.scheduler.SynchronousSchedulerClient; -import java.net.http.HttpClient; -import java.nio.file.Path; -import org.junit.jupiter.api.Test; - -class ConfigurationApiTest { - - @Test - void testImportDefinitions() { - final Configs configs = mock(Configs.class); - when(configs.getAirbyteVersion()).thenReturn(new AirbyteVersion("0.1.0-alpha")); - when(configs.getWebappUrl()).thenReturn("http://localhost"); - - final ConfigurationApi configurationApi = new ConfigurationApi( - mock(ConfigRepository.class), - mock(JobPersistence.class), - mock(SecretsRepositoryReader.class), - mock(SecretsRepositoryWriter.class), - mock(SynchronousSchedulerClient.class), - mock(StatePersistence.class), - mock(TrackingClient.class), - WorkerEnvironment.DOCKER, - LogConfigs.EMPTY, - new AirbyteVersion("0.1.0-alpha"), - Path.of(""), - mock(HttpClient.class), - mock(EventRunner.class)); - - assertFalse(configurationApi.getHealthCheck().getAvailable()); - } - -} diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java new file mode 100644 index 000000000000..7267cbd86876 --- /dev/null +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.api.model.generated.HealthCheckRead; +import io.airbyte.server.handlers.HealthCheckHandler; +import org.junit.jupiter.api.Test; + +class HealthCheckApiTest { + + @Test + void testImportDefinitions() { + final HealthCheckHandler healthCheckHandler = mock(HealthCheckHandler.class); + when(healthCheckHandler.health()) + .thenReturn(new HealthCheckRead().available( + false + )); + + final HealthApiController configurationApi = new HealthApiController(healthCheckHandler); + + assertFalse(configurationApi.getHealthCheck().getAvailable()); + } + +} From 293c18779e9c7c06b1cfcca4bcfc0e39456f9c2b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Thu, 27 Oct 2022 10:58:13 -0400 Subject: [PATCH 25/34] format --- .../test/java/io/airbyte/server/apis/HealthCheckApiTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java index 7267cbd86876..cc4f5d814e35 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java @@ -19,8 +19,7 @@ void testImportDefinitions() { final HealthCheckHandler healthCheckHandler = mock(HealthCheckHandler.class); when(healthCheckHandler.health()) .thenReturn(new HealthCheckRead().available( - false - )); + false)); final HealthApiController configurationApi = new HealthApiController(healthCheckHandler); From 7accc25b94a1b52669ece193274c3b8ee24fe661 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 28 Oct 2022 11:26:22 -0400 Subject: [PATCH 26/34] Extract logs api --- .../java/io/airbyte/server/ServerFactory.java | 14 ++++++++++-- .../airbyte/server/apis/ConfigurationApi.java | 9 ++++---- .../server/apis/LogsApiController.java | 17 ++++++++++++++ .../server/apis/binders/LogsApiBinder.java | 16 ++++++++++++++ .../server/apis/factories/LogsApiFactory.java | 22 +++++++++++++++++++ .../airbyte/server/handlers/LogsHandler.java | 17 +++++++------- .../server/handlers/LogsHandlerTest.java | 6 ++--- 7 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 5ecc963db3c6..bc3fca244a52 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -22,6 +22,7 @@ import io.airbyte.server.apis.DestinationDefinitionApiController; import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; import io.airbyte.server.apis.HealthApiController; +import io.airbyte.server.apis.LogsApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.binders.DbMigrationBinder; @@ -29,6 +30,7 @@ import io.airbyte.server.apis.binders.DestinationDefinitionApiBinder; import io.airbyte.server.apis.binders.DestinationDefinitionSpecificationApiBinder; import io.airbyte.server.apis.binders.HealthApiBinder; +import io.airbyte.server.apis.binders.LogsApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; @@ -37,6 +39,7 @@ import io.airbyte.server.apis.factories.DestinationDefinitionSpecificationApiFactory; import io.airbyte.server.apis.factories.HealthApiFactory; import io.airbyte.server.apis.factories.JobsApiFactory; +import io.airbyte.server.apis.factories.LogsApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; @@ -44,6 +47,7 @@ import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.HealthCheckHandler; import io.airbyte.server.handlers.JobHistoryHandler; +import io.airbyte.server.handlers.LogsHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.scheduler.EventRunner; @@ -80,6 +84,7 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien final DestinationHandler destinationApiHandler, final HealthCheckHandler healthCheckHandler, final JobHistoryHandler jobHistoryHandler, + final LogsHandler logsHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler); @@ -109,6 +114,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final DestinationHandler destinationApiHandler, final HealthCheckHandler healthCheckHandler, final JobHistoryHandler jobHistoryHandler, + final LogsHandler logsHandler, final OperationsHandler operationsHandler, final SchedulerHandler schedulerHandler) { final Map mdc = MDC.getCopyOfContextMap(); @@ -154,6 +160,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul JobsApiFactory.setValues(jobHistoryHandler, schedulerHandler); + LogsApiFactory.setValues(logsHandler); + // server configurations final Set> componentClasses = Set.of( ConfigurationApi.class, @@ -163,7 +171,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DestinationApiController.class, DestinationDefinitionApiController.class, DestinationDefinitionSpecificationApiController.class, - HealthApiController.class); + HealthApiController.class, + LogsApiController.class); final Set components = Set.of( new CorsFilter(), @@ -174,7 +183,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul new DestinationApiBinder(), new DestinationDefinitionApiBinder(), new DestinationDefinitionSpecificationApiBinder(), - new HealthApiBinder()); + new HealthApiBinder(), + new LogsApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 6eab18d1998c..6684a1cee5ff 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -117,7 +117,6 @@ import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.JobHistoryHandler; -import io.airbyte.server.handlers.LogsHandler; import io.airbyte.server.handlers.OAuthHandler; import io.airbyte.server.handlers.OpenApiConfigHandler; import io.airbyte.server.handlers.OperationsHandler; @@ -156,7 +155,6 @@ public class ConfigurationApi implements io.airbyte.api.generated.V1Api { private final JobHistoryHandler jobHistoryHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; private final WebBackendGeographiesHandler webBackendGeographiesHandler; - private final LogsHandler logsHandler; private final OpenApiConfigHandler openApiConfigHandler; private final OAuthHandler oAuthHandler; private final WorkerEnvironment workerEnvironment; @@ -235,7 +233,6 @@ public ConfigurationApi(final ConfigRepository configRepository, eventRunner, configRepository); webBackendGeographiesHandler = new WebBackendGeographiesHandler(); - logsHandler = new LogsHandler(); openApiConfigHandler = new OpenApiConfigHandler(); } @@ -945,9 +942,13 @@ public AttemptNormalizationStatusReadList getAttemptNormalizationStatusesForJob( return execute(() -> jobHistoryHandler.getAttemptNormalizationStatuses(jobIdRequestBody)); } + /** + * This implementation has been moved to {@link LogsApiController}. Since the path of + * {@link LogsApiController} is more granular, it will override this implementation + */ @Override public File getLogs(final LogsRequestBody logsRequestBody) { - return execute(() -> logsHandler.getLogs(workspaceRoot, workerEnvironment, logConfigs, logsRequestBody)); + throw new NotImplementedException(); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java new file mode 100644 index 000000000000..5bece5321bba --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java @@ -0,0 +1,17 @@ +package io.airbyte.server.apis; + +import io.airbyte.api.generated.LogsApi; +import io.airbyte.api.model.generated.LogsRequestBody; +import io.airbyte.server.handlers.LogsHandler; +import java.io.File; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class LogsApiController implements LogsApi { + + private final LogsHandler logsHandler; + + @Override public File getLogs(final LogsRequestBody logsRequestBody) { + return ConfigurationApi.execute(() -> logsHandler.getLogs(logsRequestBody)); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java new file mode 100644 index 000000000000..50f28a470091 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java @@ -0,0 +1,16 @@ +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.LogsApiController; +import io.airbyte.server.apis.factories.LogsApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class LogsApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(LogsApiFactory.class) + .to(LogsApiController.class) + .in(RequestScoped.class); + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java new file mode 100644 index 000000000000..b160a3e6678f --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java @@ -0,0 +1,22 @@ +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.LogsApiController; +import io.airbyte.server.handlers.LogsHandler; +import org.glassfish.hk2.api.Factory; + +public class LogsApiFactory implements Factory { + + private static LogsHandler logsHandler; + + public static void setValues(final LogsHandler logsHandler) { + LogsApiFactory.logsHandler = logsHandler; + } + + @Override public LogsApiController provide() { + return new LogsApiController(logsHandler); + } + + @Override public void dispose(final LogsApiController instance) { + /* no op */ + } +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java index 1b3f581aec09..5b6a360d061a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java @@ -5,28 +5,27 @@ package io.airbyte.server.handlers; import io.airbyte.api.model.generated.LogsRequestBody; -import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.Configs; import io.airbyte.config.helpers.LogClientSingleton; -import io.airbyte.config.helpers.LogConfigs; import java.io.File; -import java.nio.file.Path; +import lombok.AllArgsConstructor; /** * This handler is only responsible for server and scheduler logs. Jobs logs paths are determined by * the submitJob function in the JobSubmitter class in the airbyte-server module. */ +@AllArgsConstructor public class LogsHandler { - public File getLogs(final Path workspaceRoot, - final WorkerEnvironment workerEnvironment, - final LogConfigs logConfigs, - final LogsRequestBody logsRequestBody) { + private final Configs configs; + + public File getLogs(final LogsRequestBody logsRequestBody) { switch (logsRequestBody.getLogType()) { case SERVER -> { - return LogClientSingleton.getInstance().getServerLogFile(workspaceRoot, workerEnvironment, logConfigs); + return LogClientSingleton.getInstance().getServerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), configs.getLogConfigs()); } case SCHEDULER -> { - return LogClientSingleton.getInstance().getSchedulerLogFile(workspaceRoot, workerEnvironment, logConfigs); + return LogClientSingleton.getInstance().getSchedulerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), configs.getLogConfigs()); } default -> throw new IllegalStateException("Unexpected value: " + logsRequestBody.getLogType()); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/LogsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/LogsHandlerTest.java index 3ae3065d7549..098f05f1a4a3 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/LogsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/LogsHandlerTest.java @@ -28,8 +28,7 @@ void testServerLogs() { when(configs.getLogConfigs()).thenReturn(LogConfigs.EMPTY); final File expected = Path.of(String.format("/workspace/server/logs/%s", LogClientSingleton.LOG_FILENAME)).toFile(); - final File actual = new LogsHandler().getLogs(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), - configs.getLogConfigs(), new LogsRequestBody().logType(LogType.SERVER)); + final File actual = new LogsHandler(configs).getLogs(new LogsRequestBody().logType(LogType.SERVER)); assertEquals(expected, actual); } @@ -42,8 +41,7 @@ void testSchedulerLogs() { when(configs.getLogConfigs()).thenReturn(LogConfigs.EMPTY); final File expected = Path.of(String.format("/workspace/scheduler/logs/%s", LogClientSingleton.LOG_FILENAME)).toFile(); - final File actual = new LogsHandler().getLogs(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), - configs.getLogConfigs(), new LogsRequestBody().logType(LogType.SCHEDULER)); + final File actual = new LogsHandler(configs).getLogs(new LogsRequestBody().logType(LogType.SCHEDULER)); assertEquals(expected, actual); } From 5151909cda370d8d347c24223ad5eeee9135bf0b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 28 Oct 2022 11:27:40 -0400 Subject: [PATCH 27/34] Add missing declaration --- .../src/main/java/io/airbyte/server/ServerFactory.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index 5ecc963db3c6..4e8723a5cb00 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -22,6 +22,7 @@ import io.airbyte.server.apis.DestinationDefinitionApiController; import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; import io.airbyte.server.apis.HealthApiController; +import io.airbyte.server.apis.JobsApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.binders.DbMigrationBinder; @@ -29,6 +30,7 @@ import io.airbyte.server.apis.binders.DestinationDefinitionApiBinder; import io.airbyte.server.apis.binders.DestinationDefinitionSpecificationApiBinder; import io.airbyte.server.apis.binders.HealthApiBinder; +import io.airbyte.server.apis.binders.JobsApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; @@ -163,7 +165,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DestinationApiController.class, DestinationDefinitionApiController.class, DestinationDefinitionSpecificationApiController.class, - HealthApiController.class); + HealthApiController.class, + JobsApiController.class); final Set components = Set.of( new CorsFilter(), @@ -174,7 +177,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul new DestinationApiBinder(), new DestinationDefinitionApiBinder(), new DestinationDefinitionSpecificationApiBinder(), - new HealthApiBinder()); + new HealthApiBinder(), + new JobsApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); From ffb770b07a064d4aa9239beffe1a70f84bdbdecb Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 28 Oct 2022 11:52:39 -0400 Subject: [PATCH 28/34] Fix build --- airbyte-server/src/main/java/io/airbyte/server/ServerApp.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 082ee1221f5c..84b0ae284424 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -61,6 +61,7 @@ import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.HealthCheckHandler; import io.airbyte.server.handlers.JobHistoryHandler; +import io.airbyte.server.handlers.LogsHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.handlers.SourceDefinitionsHandler; @@ -325,6 +326,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, destinationDefinitionsHandler, configs.getAirbyteVersion()); + final LogsHandler logsHandler = new LogsHandler(configs); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -351,6 +354,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, destinationHandler, healthCheckHandler, jobHistoryHandler, + logsHandler, operationsHandler, schedulerHandler); } From 2142e2635dc9ac3bdb52dcd6c260091ad93b635b Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 28 Oct 2022 16:23:45 -0400 Subject: [PATCH 29/34] Tmp --- .../io/airbyte/server/apis/ConfigurationApi.java | 6 +++++- .../server/apis/NotificationsApiController.java | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 6684a1cee5ff..36b2ab77d567 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -284,9 +284,13 @@ public void updateWorkspaceFeedback(final WorkspaceGiveFeedback workspaceGiveFee }); } + /** + * This implementation has been moved to {@link AttemptApiController}. Since the path of + * {@link AttemptApiController} is more granular, it will override this implementation + */ @Override public NotificationRead tryNotificationConfig(final Notification notification) { - return execute(() -> workspacesHandler.tryNotification(notification)); + throw new NotImplementedException(); } // SOURCE diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java new file mode 100644 index 000000000000..d43449cc2db4 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java @@ -0,0 +1,12 @@ +package io.airbyte.server.apis; + +import io.airbyte.api.generated.NotificationsApi; +import io.airbyte.api.model.generated.Notification; +import io.airbyte.api.model.generated.NotificationRead; + +public class NotificationsApiController implements NotificationsApi { + + @Override public NotificationRead tryNotificationConfig(final Notification notification) { + return ConfigurationApi.execute(() -> workspacesHandler.tryNotification(notification)); + } +} From 3ed2fee43513e189b270d7d55b02f99c199ab9e4 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Fri, 28 Oct 2022 16:25:36 -0400 Subject: [PATCH 30/34] format and PR comments --- .../io/airbyte/server/apis/LogsApiController.java | 10 +++++++++- .../io/airbyte/server/apis/binders/LogsApiBinder.java | 5 +++++ .../airbyte/server/apis/factories/LogsApiFactory.java | 11 +++++++++-- .../java/io/airbyte/server/handlers/LogsHandler.java | 3 ++- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java index 5bece5321bba..b3c1e5f58619 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java @@ -1,17 +1,25 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis; import io.airbyte.api.generated.LogsApi; import io.airbyte.api.model.generated.LogsRequestBody; import io.airbyte.server.handlers.LogsHandler; import java.io.File; +import javax.ws.rs.Path; import lombok.AllArgsConstructor; +@Path("/v1/logs/get") @AllArgsConstructor public class LogsApiController implements LogsApi { private final LogsHandler logsHandler; - @Override public File getLogs(final LogsRequestBody logsRequestBody) { + @Override + public File getLogs(final LogsRequestBody logsRequestBody) { return ConfigurationApi.execute(() -> logsHandler.getLogs(logsRequestBody)); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java index 50f28a470091..037597e9e5f1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.binders; import io.airbyte.server.apis.LogsApiController; @@ -13,4 +17,5 @@ protected void configure() { .to(LogsApiController.class) .in(RequestScoped.class); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java index b160a3e6678f..450419d6a515 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.factories; import io.airbyte.server.apis.LogsApiController; @@ -12,11 +16,14 @@ public static void setValues(final LogsHandler logsHandler) { LogsApiFactory.logsHandler = logsHandler; } - @Override public LogsApiController provide() { + @Override + public LogsApiController provide() { return new LogsApiController(logsHandler); } - @Override public void dispose(final LogsApiController instance) { + @Override + public void dispose(final LogsApiController instance) { /* no op */ } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java index 5b6a360d061a..eacb4760bfb3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java @@ -25,7 +25,8 @@ public File getLogs(final LogsRequestBody logsRequestBody) { return LogClientSingleton.getInstance().getServerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), configs.getLogConfigs()); } case SCHEDULER -> { - return LogClientSingleton.getInstance().getSchedulerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), configs.getLogConfigs()); + return LogClientSingleton.getInstance().getSchedulerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), + configs.getLogConfigs()); } default -> throw new IllegalStateException("Unexpected value: " + logsRequestBody.getLogType()); } From 9b00d891f3931193dba5e24204efc4f92bea24b3 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 1 Nov 2022 16:14:25 -0400 Subject: [PATCH 31/34] Extract notification API --- .../java/io/airbyte/server/ServerApp.java | 11 ++++++- .../java/io/airbyte/server/ServerFactory.java | 18 +++++++++--- .../server/apis/LogsApiController.java | 8 ++++- .../apis/NotificationsApiController.java | 15 +++++++++- .../server/apis/binders/LogsApiBinder.java | 5 ++++ .../apis/binders/NotificationApiBinder.java | 21 ++++++++++++++ .../server/apis/factories/LogsApiFactory.java | 11 +++++-- .../factories/NotificationsApiFactory.java | 29 +++++++++++++++++++ .../airbyte/server/handlers/LogsHandler.java | 3 +- 9 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/binders/NotificationApiBinder.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/factories/NotificationsApiFactory.java diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 84b0ae284424..9a64e6f5d310 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -66,6 +66,7 @@ import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.handlers.SourceDefinitionsHandler; import io.airbyte.server.handlers.SourceHandler; +import io.airbyte.server.handlers.WorkspacesHandler; import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.TemporalEventRunner; @@ -328,6 +329,13 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final LogsHandler logsHandler = new LogsHandler(configs); + final WorkspacesHandler workspacesHandler = new WorkspacesHandler( + configRepository, + secretsRepositoryWriter, + connectionsHandler, + destinationHandler, + sourceHandler); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -356,7 +364,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, jobHistoryHandler, logsHandler, operationsHandler, - schedulerHandler); + schedulerHandler, + workspacesHandler); } @VisibleForTesting diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index dccdda8604d6..53343cee235e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -24,6 +24,7 @@ import io.airbyte.server.apis.HealthApiController; import io.airbyte.server.apis.JobsApiController; import io.airbyte.server.apis.LogsApiController; +import io.airbyte.server.apis.NotificationsApiController; import io.airbyte.server.apis.binders.AttemptApiBinder; import io.airbyte.server.apis.binders.ConnectionApiBinder; import io.airbyte.server.apis.binders.DbMigrationBinder; @@ -33,6 +34,7 @@ import io.airbyte.server.apis.binders.HealthApiBinder; import io.airbyte.server.apis.binders.JobsApiBinder; import io.airbyte.server.apis.binders.LogsApiBinder; +import io.airbyte.server.apis.binders.NotificationApiBinder; import io.airbyte.server.apis.factories.AttemptApiFactory; import io.airbyte.server.apis.factories.ConnectionApiFactory; import io.airbyte.server.apis.factories.DbMigrationApiFactory; @@ -42,6 +44,7 @@ import io.airbyte.server.apis.factories.HealthApiFactory; import io.airbyte.server.apis.factories.JobsApiFactory; import io.airbyte.server.apis.factories.LogsApiFactory; +import io.airbyte.server.apis.factories.NotificationsApiFactory; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; @@ -52,6 +55,7 @@ import io.airbyte.server.handlers.LogsHandler; import io.airbyte.server.handlers.OperationsHandler; import io.airbyte.server.handlers.SchedulerHandler; +import io.airbyte.server.handlers.WorkspacesHandler; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; @@ -88,7 +92,8 @@ ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClien final JobHistoryHandler jobHistoryHandler, final LogsHandler logsHandler, final OperationsHandler operationsHandler, - final SchedulerHandler schedulerHandler); + final SchedulerHandler schedulerHandler, + final WorkspacesHandler workspacesHandler); class Api implements ServerFactory { @@ -118,7 +123,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final JobHistoryHandler jobHistoryHandler, final LogsHandler logsHandler, final OperationsHandler operationsHandler, - final SchedulerHandler schedulerHandler) { + final SchedulerHandler schedulerHandler, + final WorkspacesHandler workspacesHandler) { final Map mdc = MDC.getCopyOfContextMap(); // set static values for factory @@ -164,6 +170,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul LogsApiFactory.setValues(logsHandler); + NotificationsApiFactory.setValues(workspacesHandler); + // server configurations final Set> componentClasses = Set.of( ConfigurationApi.class, @@ -175,7 +183,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul DestinationDefinitionSpecificationApiController.class, HealthApiController.class, JobsApiController.class, - LogsApiController.class); + LogsApiController.class, + NotificationsApiController.class); final Set components = Set.of( new CorsFilter(), @@ -188,7 +197,8 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul new DestinationDefinitionSpecificationApiBinder(), new HealthApiBinder(), new JobsApiBinder(), - new LogsApiBinder()); + new LogsApiBinder(), + new NotificationApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java index 5bece5321bba..c628e4cbed6a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis; import io.airbyte.api.generated.LogsApi; @@ -11,7 +15,9 @@ public class LogsApiController implements LogsApi { private final LogsHandler logsHandler; - @Override public File getLogs(final LogsRequestBody logsRequestBody) { + @Override + public File getLogs(final LogsRequestBody logsRequestBody) { return ConfigurationApi.execute(() -> logsHandler.getLogs(logsRequestBody)); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java index d43449cc2db4..e6f6df3b6370 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotificationsApiController.java @@ -1,12 +1,25 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis; import io.airbyte.api.generated.NotificationsApi; import io.airbyte.api.model.generated.Notification; import io.airbyte.api.model.generated.NotificationRead; +import io.airbyte.server.handlers.WorkspacesHandler; +import javax.ws.rs.Path; +import lombok.AllArgsConstructor; +@Path("/v1/notifications/try") +@AllArgsConstructor public class NotificationsApiController implements NotificationsApi { - @Override public NotificationRead tryNotificationConfig(final Notification notification) { + private final WorkspacesHandler workspacesHandler; + + @Override + public NotificationRead tryNotificationConfig(final Notification notification) { return ConfigurationApi.execute(() -> workspacesHandler.tryNotification(notification)); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java index 50f28a470091..037597e9e5f1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/LogsApiBinder.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.binders; import io.airbyte.server.apis.LogsApiController; @@ -13,4 +17,5 @@ protected void configure() { .to(LogsApiController.class) .in(RequestScoped.class); } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/NotificationApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/NotificationApiBinder.java new file mode 100644 index 000000000000..009c3e2d48b5 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/NotificationApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.NotificationsApiController; +import io.airbyte.server.apis.factories.NotificationsApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class NotificationApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(NotificationsApiFactory.class) + .to(NotificationsApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java index b160a3e6678f..450419d6a515 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/LogsApiFactory.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.server.apis.factories; import io.airbyte.server.apis.LogsApiController; @@ -12,11 +16,14 @@ public static void setValues(final LogsHandler logsHandler) { LogsApiFactory.logsHandler = logsHandler; } - @Override public LogsApiController provide() { + @Override + public LogsApiController provide() { return new LogsApiController(logsHandler); } - @Override public void dispose(final LogsApiController instance) { + @Override + public void dispose(final LogsApiController instance) { /* no op */ } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/NotificationsApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/NotificationsApiFactory.java new file mode 100644 index 000000000000..29a3cf7e9096 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/NotificationsApiFactory.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.NotificationsApiController; +import io.airbyte.server.handlers.WorkspacesHandler; +import org.glassfish.hk2.api.Factory; + +public class NotificationsApiFactory implements Factory { + + private static WorkspacesHandler workspacesHandler; + + public static void setValues(final WorkspacesHandler workspacesHandler) { + NotificationsApiFactory.workspacesHandler = workspacesHandler; + } + + @Override + public NotificationsApiController provide() { + return new NotificationsApiController(NotificationsApiFactory.workspacesHandler); + } + + @Override + public void dispose(final NotificationsApiController instance) { + /* no op */ + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java index 5b6a360d061a..eacb4760bfb3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java @@ -25,7 +25,8 @@ public File getLogs(final LogsRequestBody logsRequestBody) { return LogClientSingleton.getInstance().getServerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), configs.getLogConfigs()); } case SCHEDULER -> { - return LogClientSingleton.getInstance().getSchedulerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), configs.getLogConfigs()); + return LogClientSingleton.getInstance().getSchedulerLogFile(configs.getWorkspaceRoot(), configs.getWorkerEnvironment(), + configs.getLogConfigs()); } default -> throw new IllegalStateException("Unexpected value: " + logsRequestBody.getLogType()); } From 23d908a1acd12bf3504d75fec1357b904f125ada Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 1 Nov 2022 16:49:45 -0400 Subject: [PATCH 32/34] re-organize tags --- airbyte-api/src/main/openapi/config.yaml | 18 +- .../api/generated-api-html/index.html | 712 +++++++++--------- 2 files changed, 368 insertions(+), 362 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 69adc4af4cb6..a787b3b63353 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -49,8 +49,10 @@ tags: description: Destination related resources. - name: connection description: Connection between sources and destinations. - - name: oauth - description: OAuth related resources to delegate access from user. + - name: destination_oauth + description: Source OAuth related resources to delegate access from user. + - name: source_oauth + description: Source OAuth related resources to delegate access from user. - name: db_migration description: Database migration related resources. - name: web_backend @@ -1762,7 +1764,7 @@ paths: /v1/source_oauths/oauth_params/create: post: tags: - - oauth + - source_oauth summary: > Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with @@ -1785,7 +1787,7 @@ paths: /v1/source_oauths/get_consent_url: post: tags: - - oauth + - source_oauth summary: Given a source connector definition ID, return the URL to the consent screen where to redirect the user to. operationId: getSourceOAuthConsent requestBody: @@ -1808,7 +1810,7 @@ paths: /v1/source_oauths/complete_oauth: post: tags: - - oauth + - source_oauth summary: Given a source def ID generate an access/refresh token etc. operationId: completeSourceOAuth requestBody: @@ -1831,7 +1833,7 @@ paths: /v1/destination_oauths/get_consent_url: post: tags: - - oauth + - destination_oauth summary: Given a destination connector definition ID, return the URL to the consent screen where to redirect the user to. operationId: getDestinationOAuthConsent requestBody: @@ -1854,7 +1856,7 @@ paths: /v1/destination_oauths/complete_oauth: post: tags: - - oauth + - destination_oauth summary: Given a destination def ID generate an access/refresh token etc. operationId: completeDestinationOAuth requestBody: @@ -1877,7 +1879,7 @@ paths: /v1/destination_oauths/oauth_params/create: post: tags: - - oauth + - destination_oauth summary: > Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 92bd602330f3..19a3bd5f9fec 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -273,6 +273,12 @@

DestinationDefinitionSpecifica +

DestinationOauth

+

Health