diff --git a/.github/labeler.yml b/.github/labeler.yml index efdbb9f069f8..c24ac0551bf9 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -4,12 +4,12 @@ area/platform: - airbyte-webapp/**/* - airbyte-api/* - airbyte-api/**/* - - airbyte-server/* - - airbyte-server/**/* - - airbyte-scheduler/* - - airbyte-scheduler/**/* + - airbyte-persistence/* + - airbyte-persistence/**/* - airbyte-protocol/* - airbyte-protocol/**/* + - airbyte-server/* + - airbyte-server/**/* - airbyte-workers/* - airbyte-workers/**/* - kube/* @@ -29,10 +29,6 @@ area/server: - airbyte-server/* - airbyte-server/**/* -area/scheduler: - - airbyte-scheduler/* - - airbyte-scheduler/**/* - area/protocol: - airbyte-protocol/* - airbyte-protocol/**/* diff --git a/airbyte-config/config-persistence/readme.md b/airbyte-config/config-persistence/readme.md index df65ce284cea..0bfd8b20dcfd 100644 --- a/airbyte-config/config-persistence/readme.md +++ b/airbyte-config/config-persistence/readme.md @@ -1,6 +1,6 @@ # airbyte-config:config-persistence -This module contains the logic for accessing the config database. This database is primarily used by the `airbyte-server` but is also accessed from `airbyte-scheduler` and `airbyte-workers`. It contains all configuration information for Airbyte. +This module contains the logic for accessing the config database. This database is primarily used by the `airbyte-server` but is also accessed from `airbyte-workers`. It contains all configuration information for Airbyte. ## Key files * `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java` The only other one that is used is the `YamlSeedConfigPersistence.java` which is used for loading configs that ship with the app. diff --git a/airbyte-container-orchestrator/build.gradle b/airbyte-container-orchestrator/build.gradle index b4bbf9431aef..4695dc4b4d92 100644 --- a/airbyte-container-orchestrator/build.gradle +++ b/airbyte-container-orchestrator/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-json-validation') implementation project(':airbyte-protocol:protocol-models') - implementation project(':airbyte-scheduler:scheduler-persistence') + implementation project(':airbyte-persistence:job-persistence') implementation project(':airbyte-workers') implementation project(':airbyte-metrics:metrics-lib') diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java index d7c55cd518c1..198e977d71ea 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java @@ -12,7 +12,7 @@ * * Application Name Conventions: *

- * - Drop the airbyte prefix when naming applications, e.g airbyte-scheduler -> scheduler. + * - Drop the airbyte prefix when naming applications, e.g airbyte-server -> server. *

* - Use dashes to delimit application names with multiple words. *

diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java index 22ccba1dac08..d706f8e5d565 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java @@ -551,7 +551,7 @@ void testYamlSchemas() throws IOException { final String tableName = tableSchema.name(); final JsonNode schema = tableSchema.getTableDefinition(); assertNotNull(schema, - "Json schema files should be created in airbyte-scheduler/src/main/resources/tables for every table in the Database to validate its content"); + "Json schema files should be created in airbyte-persistence/job-persistence/src/main/resources/tables for every table in the Database to validate its content"); tableStream.forEach(row -> { try { jsonSchemaValidator.ensure(schema, row); diff --git a/airbyte-scheduler/client/build.gradle b/airbyte-scheduler/client/build.gradle deleted file mode 100644 index 36b424f1d49f..000000000000 --- a/airbyte-scheduler/client/build.gradle +++ /dev/null @@ -1,21 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-config:specs') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-protocol:protocol-models') - implementation project(':airbyte-scheduler:scheduler-models') - implementation project(':airbyte-scheduler:scheduler-persistence') - implementation project(':airbyte-persistence:job-persistence') - // todo (cgardens) - remove this dep. just needs temporal client. - implementation project(':airbyte-workers') - - implementation 'com.google.cloud:google-cloud-storage:2.2.2' - -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-scheduler/client/readme.md b/airbyte-scheduler/client/readme.md deleted file mode 100644 index a9e529fd5203..000000000000 --- a/airbyte-scheduler/client/readme.md +++ /dev/null @@ -1,6 +0,0 @@ -# airbyte-scheduler:client - -Java clients for submitting Jobs. - -## Key Files -* `SynchronousSchedulerClient` - interface for scheduling _synchronous_ jobs. diff --git a/airbyte-scheduler/scheduler-models/build.gradle b/airbyte-scheduler/scheduler-models/build.gradle deleted file mode 100644 index 55795d2cd1d3..000000000000 --- a/airbyte-scheduler/scheduler-models/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-scheduler/scheduler-models/readme.md b/airbyte-scheduler/scheduler-models/readme.md deleted file mode 100644 index decf60fb860b..000000000000 --- a/airbyte-scheduler/scheduler-models/readme.md +++ /dev/null @@ -1,8 +0,0 @@ -# airbyte-scheduler:scheduler-models - -This module declares models that belong to the Scheduler. - -These models are generated in the same way as `airbyte-config:models`. See that module for reference. - -## Key Files -* `Job.java` and `Attempt.java` are the top-level models in this schema. diff --git a/airbyte-scheduler/scheduler-persistence/build.gradle b/airbyte-scheduler/scheduler-persistence/build.gradle deleted file mode 100644 index c40c4355a6ae..000000000000 --- a/airbyte-scheduler/scheduler-persistence/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation 'io.sentry:sentry:6.1.0' - - implementation project(':airbyte-analytics') - implementation project(':airbyte-commons-docker') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-db:jooq') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-notification') - implementation project(':airbyte-oauth') - implementation project(':airbyte-protocol:protocol-models') - implementation project(':airbyte-scheduler:scheduler-models') - - testImplementation libs.flyway.core - testImplementation libs.platform.testcontainers.postgresql - testImplementation project(':airbyte-test-utils') -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-scheduler/scheduler-persistence/readme.md b/airbyte-scheduler/scheduler-persistence/readme.md deleted file mode 100644 index 0c61afb6cd6c..000000000000 --- a/airbyte-scheduler/scheduler-persistence/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# airbyte-scheduler:scheduler-persistence - -This module encapsulates the logic for the Jobs Database. This Database is primarily used by the `airbyte-scheduler` and `airbyte-workers` but it is also access from the `airbyte-server`. - -## Key Files -* `DefaultJobPersistence` is where all queries for interacting with the Jobs Database live. -* everything else is abstraction on top of that to make it easier to create / interact with / test jobs. diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index e08e809fd26b..af56de876b02 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -15,9 +15,6 @@ dependencies { implementation project(':airbyte-notification') implementation project(':airbyte-oauth') implementation project(':airbyte-protocol:protocol-models') - implementation(project(':airbyte-scheduler:client')) { - exclude module: 'airbyte-workers' - } implementation project(':airbyte-persistence:job-persistence') implementation(project(':airbyte-workers')) { // Temporary hack to avoid dependency conflicts diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java index f417e8428b32..bc2b2703ae8f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java @@ -16,9 +16,9 @@ import io.airbyte.config.persistence.StatePersistence; import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.scheduler.client.EventRunner; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.apis.ConfigurationApi; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Map; diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index aa3d7eeec519..bbb5e4c45bf8 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -44,9 +44,6 @@ import io.airbyte.persistence.job.errorreporter.JobErrorReportingClientFactory; import io.airbyte.persistence.job.factory.OAuthConfigSupplier; import io.airbyte.persistence.job.tracker.JobTracker; -import io.airbyte.scheduler.client.DefaultSynchronousSchedulerClient; -import io.airbyte.scheduler.client.EventRunner; -import io.airbyte.scheduler.client.TemporalEventRunner; import io.airbyte.server.errors.InvalidInputExceptionMapper; import io.airbyte.server.errors.InvalidJsonExceptionMapper; import io.airbyte.server.errors.InvalidJsonInputExceptionMapper; @@ -54,6 +51,9 @@ import io.airbyte.server.errors.NotFoundExceptionMapper; import io.airbyte.server.errors.UncaughtExceptionMapper; import io.airbyte.server.handlers.DbMigrationHandler; +import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.TemporalEventRunner; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.airbyte.workers.temporal.ConnectionManagerUtils; diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index bbe839251550..aa0af731c63e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -16,9 +16,9 @@ import io.airbyte.config.persistence.StatePersistence; import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.scheduler.client.EventRunner; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.apis.ConfigurationApi; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Set; diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 88fccc0a5559..3b3ebd0e85ff 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -113,8 +113,6 @@ import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.WorkspaceHelper; -import io.airbyte.scheduler.client.EventRunner; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.errors.BadObjectSchemaKnownException; import io.airbyte.server.errors.IdNotFoundKnownException; import io.airbyte.server.handlers.ArchiveHandler; @@ -135,6 +133,8 @@ import io.airbyte.server.handlers.StateHandler; import io.airbyte.server.handlers.WebBackendConnectionsHandler; import io.airbyte.server.handlers.WorkspacesHandler; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import java.io.File; diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java index 7366b7496087..614344230532 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java @@ -39,8 +39,8 @@ import io.airbyte.config.helpers.LogConfigs; import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.Job; -import io.airbyte.scheduler.client.SynchronousJobMetadata; -import io.airbyte.scheduler.client.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousJobMetadata; +import io.airbyte.server.scheduler.SynchronousResponse; import io.airbyte.workers.helper.ProtocolConverters; import java.io.IOException; import java.nio.file.Path; diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/SpecFetcher.java b/airbyte-server/src/main/java/io/airbyte/server/converters/SpecFetcher.java index 40ea57edc857..5934e6fb7756 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/SpecFetcher.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/SpecFetcher.java @@ -6,7 +6,7 @@ import com.google.common.base.Preconditions; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.scheduler.client.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousResponse; public class SpecFetcher { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java index 8e21a4bc9167..6259e3342b15 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java @@ -43,7 +43,6 @@ import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.scheduler.client.EventRunner; import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.converters.CatalogDiffConverters; import io.airbyte.server.handlers.helpers.CatalogConverter; @@ -51,6 +50,7 @@ import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper; import io.airbyte.server.handlers.helpers.DestinationMatcher; import io.airbyte.server.handlers.helpers.SourceMatcher; +import io.airbyte.server.scheduler.EventRunner; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.helper.ConnectionHelper; import java.io.IOException; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index 253412a8a600..bf64ceeb271c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -30,12 +30,12 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.scheduler.client.SynchronousResponse; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.server.errors.IdNotFoundKnownException; import io.airbyte.server.errors.InternalServerKnownException; +import io.airbyte.server.scheduler.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.server.services.AirbyteGithubStore; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index e4b53cc466aa..23e54a184a1f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -52,15 +52,15 @@ import io.airbyte.persistence.job.models.Job; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.scheduler.client.EventRunner; -import io.airbyte.scheduler.client.SynchronousJobMetadata; -import io.airbyte.scheduler.client.SynchronousResponse; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.converters.ConfigurationUpdate; import io.airbyte.server.converters.JobConverter; import io.airbyte.server.converters.OauthModelConverter; import io.airbyte.server.errors.ValueConflictKnownException; import io.airbyte.server.handlers.helpers.CatalogConverter; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousJobMetadata; +import io.airbyte.server.scheduler.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.ErrorCode; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java index bf462736a4d0..027eb3c3c4f4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java @@ -31,12 +31,12 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.scheduler.client.SynchronousResponse; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.server.errors.IdNotFoundKnownException; import io.airbyte.server.errors.InternalServerKnownException; +import io.airbyte.server.scheduler.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.server.services.AirbyteGithubStore; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index 201ead8d0370..8a1e8175460e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -47,8 +47,8 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.scheduler.client.EventRunner; import io.airbyte.server.handlers.helpers.CatalogConverter; +import io.airbyte.server.scheduler.EventRunner; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.helper.ProtocolConverters; import io.airbyte.workers.temporal.TemporalClient.ManualOperationResult; diff --git a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java b/airbyte-server/src/main/java/io/airbyte/server/scheduler/DefaultSynchronousSchedulerClient.java similarity index 99% rename from airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java rename to airbyte-server/src/main/java/io/airbyte/server/scheduler/DefaultSynchronousSchedulerClient.java index b7e1fde0788f..1b5c7c4e5939 100644 --- a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java +++ b/airbyte-server/src/main/java/io/airbyte/server/scheduler/DefaultSynchronousSchedulerClient.java @@ -2,7 +2,7 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.scheduler.client; +package io.airbyte.server.scheduler; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; diff --git a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/EventRunner.java b/airbyte-server/src/main/java/io/airbyte/server/scheduler/EventRunner.java similarity index 96% rename from airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/EventRunner.java rename to airbyte-server/src/main/java/io/airbyte/server/scheduler/EventRunner.java index 07ce8bc660b8..366c44e935b9 100644 --- a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/EventRunner.java +++ b/airbyte-server/src/main/java/io/airbyte/server/scheduler/EventRunner.java @@ -2,7 +2,7 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.scheduler.client; +package io.airbyte.server.scheduler; import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.workers.temporal.TemporalClient.ManualOperationResult; diff --git a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java b/airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousJobMetadata.java similarity index 98% rename from airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java rename to airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousJobMetadata.java index 6bc67b1b1c1d..196b00971151 100644 --- a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java +++ b/airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousJobMetadata.java @@ -2,7 +2,7 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.scheduler.client; +package io.airbyte.server.scheduler; import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.workers.temporal.JobMetadata; diff --git a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java b/airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousResponse.java similarity index 98% rename from airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java rename to airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousResponse.java index 524eca3a28b1..3905a3466fed 100644 --- a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java +++ b/airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousResponse.java @@ -2,7 +2,7 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.scheduler.client; +package io.airbyte.server.scheduler; import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.workers.temporal.TemporalResponse; diff --git a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousSchedulerClient.java b/airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousSchedulerClient.java similarity index 96% rename from airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousSchedulerClient.java rename to airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousSchedulerClient.java index 6759cd044f6a..fbe900c19f9b 100644 --- a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousSchedulerClient.java +++ b/airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousSchedulerClient.java @@ -2,7 +2,7 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.scheduler.client; +package io.airbyte.server.scheduler; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; diff --git a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/TemporalEventRunner.java b/airbyte-server/src/main/java/io/airbyte/server/scheduler/TemporalEventRunner.java similarity index 97% rename from airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/TemporalEventRunner.java rename to airbyte-server/src/main/java/io/airbyte/server/scheduler/TemporalEventRunner.java index 76fc960f13b2..6d0fb0d91ce4 100644 --- a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/TemporalEventRunner.java +++ b/airbyte-server/src/main/java/io/airbyte/server/scheduler/TemporalEventRunner.java @@ -2,7 +2,7 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.scheduler.client; +package io.airbyte.server.scheduler; import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.workers.temporal.TemporalClient; diff --git a/airbyte-server/src/test/java/io/airbyte/server/ServerAppTest.java b/airbyte-server/src/test/java/io/airbyte/server/ServerAppTest.java index 22e261189eb4..11e31422b47a 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/ServerAppTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/ServerAppTest.java @@ -12,7 +12,7 @@ import io.airbyte.config.StandardSync.Status; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.scheduler.client.EventRunner; +import io.airbyte.server.scheduler.EventRunner; import java.util.List; import java.util.Set; import java.util.UUID; diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java index 69ab59e20172..af371faf2ffd 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java @@ -21,8 +21,8 @@ import io.airbyte.config.persistence.StatePersistence; import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.scheduler.client.EventRunner; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; import java.nio.file.Path; import org.flywaydb.core.Flyway; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java index fb02433781a4..ded755dcc70e 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java @@ -59,10 +59,10 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.scheduler.client.EventRunner; import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.handlers.helpers.CatalogConverter; import io.airbyte.server.helpers.ConnectionHelpers; +import io.airbyte.server.scheduler.EventRunner; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.Collections; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index d7fbc827d291..2432cd6f5973 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -37,10 +37,10 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.scheduler.client.SynchronousJobMetadata; -import io.airbyte.scheduler.client.SynchronousResponse; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.errors.IdNotFoundKnownException; +import io.airbyte.server.scheduler.SynchronousJobMetadata; +import io.airbyte.server.scheduler.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.server.services.AirbyteGithubStore; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index 265f0829d241..0ace1315d816 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -61,15 +61,15 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.scheduler.client.EventRunner; -import io.airbyte.scheduler.client.SynchronousJobMetadata; -import io.airbyte.scheduler.client.SynchronousResponse; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.converters.ConfigurationUpdate; import io.airbyte.server.converters.JobConverter; import io.airbyte.server.errors.ValueConflictKnownException; import io.airbyte.server.helpers.DestinationHelpers; import io.airbyte.server.helpers.SourceHelpers; +import io.airbyte.server.scheduler.EventRunner; +import io.airbyte.server.scheduler.SynchronousJobMetadata; +import io.airbyte.server.scheduler.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.ErrorCode; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java index 5446e2d4a17f..1f4167817c92 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java @@ -38,10 +38,10 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.scheduler.client.SynchronousJobMetadata; -import io.airbyte.scheduler.client.SynchronousResponse; -import io.airbyte.scheduler.client.SynchronousSchedulerClient; import io.airbyte.server.errors.IdNotFoundKnownException; +import io.airbyte.server.scheduler.SynchronousJobMetadata; +import io.airbyte.server.scheduler.SynchronousResponse; +import io.airbyte.server.scheduler.SynchronousSchedulerClient; import io.airbyte.server.services.AirbyteGithubStore; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index 5af72e11bd57..b3a977b1093c 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -76,12 +76,12 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.scheduler.client.EventRunner; import io.airbyte.server.helpers.ConnectionHelpers; import io.airbyte.server.helpers.DestinationDefinitionHelpers; import io.airbyte.server.helpers.DestinationHelpers; import io.airbyte.server.helpers.SourceDefinitionHelpers; import io.airbyte.server.helpers.SourceHelpers; +import io.airbyte.server.scheduler.EventRunner; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.TemporalClient.ManualOperationResult; import java.io.IOException; diff --git a/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java b/airbyte-server/src/test/java/io/airbyte/server/scheduler/DefaultSynchronousSchedulerClientTest.java similarity index 99% rename from airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java rename to airbyte-server/src/test/java/io/airbyte/server/scheduler/DefaultSynchronousSchedulerClientTest.java index c7a2b754fb9d..79bdbaf4d2aa 100644 --- a/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/scheduler/DefaultSynchronousSchedulerClientTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.scheduler.client; +package io.airbyte.server.scheduler; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 7da3b4571283..1e93f058e9b1 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -57,7 +57,6 @@ dependencies { exclude group: 'io.micronaut.security' exclude group: 'io.micronaut.sql' } - implementation project(':airbyte-scheduler:scheduler-models') implementation project(':airbyte-api') testAnnotationProcessor platform(libs.micronaut.bom) diff --git a/build.gradle b/build.gradle index 8f026cc16d6b..c7ecac018dc0 100644 --- a/build.gradle +++ b/build.gradle @@ -538,7 +538,7 @@ publishing { from components.versionCatalog // Gradle will by default use the subproject path as the group id and the subproject name as the artifact id. - // e.g. the subproject :airbyte-scheduler:scheduler-models is imported at io.airbyte.airbyte-config:config-persistence:. + // e.g. the subproject :airbyte-example:example-models is imported at io.airbyte.airbyte-config:config-persistence:. } } repositories.add(rootProject.repositories.getByName('cloudrepo')) diff --git a/docs/deploying-airbyte/on-kubernetes.md b/docs/deploying-airbyte/on-kubernetes.md index 87433d6dff21..4325e4db3eaf 100644 --- a/docs/deploying-airbyte/on-kubernetes.md +++ b/docs/deploying-airbyte/on-kubernetes.md @@ -241,13 +241,13 @@ See the documentation for [`kubectl cp`](https://kubernetes.io/docs/reference/ge ### Listing Files ```bash -kubectl exec -it airbyte-scheduler-6b5747df5c-bj4fx ls /tmp/workspace/8 +kubectl exec -it airbyte-server-6b5747df5c-bj4fx ls /tmp/workspace/8 ``` ### Reading Files ```bash -kubectl exec -it airbyte-scheduler-6b5747df5c-bj4fx cat /tmp/workspace/8/0/logs.log +kubectl exec -it airbyte-server-6b5747df5c-bj4fx cat /tmp/workspace/8/0/logs.log ``` ### Persistent storage on GKE regional cluster diff --git a/docs/integrations/destinations/local-csv.md b/docs/integrations/destinations/local-csv.md index f0b02d405af1..37bbaf6f972a 100644 --- a/docs/integrations/destinations/local-csv.md +++ b/docs/integrations/destinations/local-csv.md @@ -51,7 +51,7 @@ The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local` If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte\_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files: -1. Access the scheduler container using `docker exec -it airbyte-scheduler bash` +1. Access the scheduler container using `docker exec -it airbyte-server bash` 2. Navigate to the default local mount using `cd /tmp/airbyte_local` 3. Navigate to the replicated file directory you specified when you created the destination, using `cd /{destination_path}` 4. List files containing the replicated data using `ls` @@ -60,7 +60,7 @@ If your Airbyte instance is running on the same computer that you are navigating You can also copy the output file to your host machine, the following command will copy the file to the current working directory you are using: ```text -docker cp airbyte-scheduler:/tmp/airbyte_local/{destination_path}/{filename}.csv . +docker cp airbyte-server:/tmp/airbyte_local/{destination_path}/{filename}.csv . ``` Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](../../operator-guides/locating-files-local-destination.md) for an alternative approach. diff --git a/docs/integrations/destinations/local-json.md b/docs/integrations/destinations/local-json.md index 8b93dfa14613..9b45da2e1eeb 100644 --- a/docs/integrations/destinations/local-json.md +++ b/docs/integrations/destinations/local-json.md @@ -51,7 +51,7 @@ The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local` If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte\_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files: -1. Access the scheduler container using `docker exec -it airbyte-scheduler bash` +1. Access the scheduler container using `docker exec -it airbyte-server bash` 2. Navigate to the default local mount using `cd /tmp/airbyte_local` 3. Navigate to the replicated file directory you specified when you created the destination, using `cd /{destination_path}` 4. List files containing the replicated data using `ls` @@ -60,7 +60,7 @@ If your Airbyte instance is running on the same computer that you are navigating You can also copy the output file to your host machine, the following command will copy the file to the current working directory you are using: ```text -docker cp airbyte-scheduler:/tmp/airbyte_local/{destination_path}/{filename}.jsonl . +docker cp airbyte-server:/tmp/airbyte_local/{destination_path}/{filename}.jsonl . ``` Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](../../operator-guides/locating-files-local-destination.md) for an alternative approach. diff --git a/docs/integrations/destinations/sqlite.md b/docs/integrations/destinations/sqlite.md index e952c004cc23..126474adcc98 100644 --- a/docs/integrations/destinations/sqlite.md +++ b/docs/integrations/destinations/sqlite.md @@ -51,7 +51,7 @@ The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local` If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte\_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files: -1. Access the scheduler container using `docker exec -it airbyte-scheduler bash` +1. Access the scheduler container using `docker exec -it airbyte-server bash` 2. Navigate to the default local mount using `cd /tmp/airbyte_local` 3. Navigate to the replicated file directory you specified when you created the destination, using `cd /{destination_path}` 4. Execute `sqlite3 {filename}` to access the data in a particular database file. @@ -59,7 +59,7 @@ If your Airbyte instance is running on the same computer that you are navigating You can also copy the output file to your host machine, the following command will copy the file to the current working directory you are using: ```text -docker cp airbyte-scheduler:/tmp/airbyte_local/{destination_path} . +docker cp airbyte-server:/tmp/airbyte_local/{destination_path} . ``` Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](../../operator-guides/locating-files-local-destination.md) for an alternative approach. diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 05deb8cc7c5f..de13686ca59c 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -66,7 +66,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers 1. In a terminal, on the host where Airbyte is running, turn off Airbyte. ```bash - kubectl delete deployments airbyte-db airbyte-scheduler airbyte-worker airbyte-server airbyte-temporal airbyte-webapp --namespace= + kubectl delete deployments airbyte-db airbyte-worker airbyte-server airbyte-temporal airbyte-webapp --namespace= ``` 2. Upgrade the kube deployment to new version. diff --git a/docs/troubleshooting/new-connection.md b/docs/troubleshooting/new-connection.md index ada2ebb126c8..9a9322707e2c 100644 --- a/docs/troubleshooting/new-connection.md +++ b/docs/troubleshooting/new-connection.md @@ -37,7 +37,7 @@ We’ve had that issue once. \(no spinner & 500 http error\). We don’t know wh You receive the error below when you tried to sync a database with a lot of tables \(6000 or more\). ```bash -airbyte-scheduler | io.grpc.StatusRuntimeException: RESOURCE_EXHAUSTED: grpc: received message larger than max ( vs. 4194304) +airbyte-server | io.grpc.StatusRuntimeException: RESOURCE_EXHAUSTED: grpc: received message larger than max ( vs. 4194304) ``` There are two Github issues tracking this problem: [Issue \#3942](https://github.com/airbytehq/airbyte/issues/3942) and [Issue \#3943](https://github.com/airbytehq/airbyte/issues/3943) diff --git a/settings.gradle b/settings.gradle index f69ef23a7126..253ecab5c9c2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -80,8 +80,6 @@ include ':airbyte-config:config-persistence' // transitively used by airbyte-wor include ':airbyte-persistence:job-persistence' // transitively used by airbyte-workers. include ':airbyte-db:jooq' // transitively used by airbyte-workers. include ':airbyte-notification' // transitively used by airbyte-workers. -include ':airbyte-scheduler:scheduler-models' // transitively used by airbyte-workers. -include ':airbyte-scheduler:scheduler-persistence' // used by airbyte-workers. // platform if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { @@ -91,7 +89,6 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD" include ':airbyte-config:specs' include ':airbyte-container-orchestrator' include ':airbyte-metrics:reporter' - include ':airbyte-scheduler:client' include ':airbyte-server' include ':airbyte-temporal' include ':airbyte-tests'