From e59e55ecbd20f712acb356a50b5d5ef44a44dabf Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Thu, 15 Sep 2022 16:25:28 -0700 Subject: [PATCH 1/4] use prototype for jwt signature because we need a new one --- .../io/airbyte/workers/config/ApiClientBeanFactory.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java index 0b258712d409..59ecf21d3f05 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java @@ -12,6 +12,7 @@ import io.airbyte.config.Configs.WorkerPlane; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; +import io.micronaut.context.annotation.Prototype; import java.io.FileInputStream; import java.security.interfaces.RSAPrivateKey; import java.util.Date; @@ -19,6 +20,8 @@ import javax.inject.Named; import javax.inject.Singleton; import lombok.extern.slf4j.Slf4j; +import io.micronaut.context.BeanProvider; + /** * Micronaut bean factory for API client singletons. @@ -33,7 +36,7 @@ public class ApiClientBeanFactory { public AirbyteApiClient airbyteApiClient( @Value("${airbyte.internal.api.auth-header.name}") final String airbyteApiAuthHeaderName, @Value("${airbyte.internal.api.host}") final String airbyteApiHost, - @Named("internalApiAuthToken") final String internalApiAuthToken, + @Named("internalApiAuthToken") final BeanProvider internalApiAuthToken, @Named("internalApiScheme") final String internalApiScheme) { return new AirbyteApiClient( new io.airbyte.api.client.invoker.generated.ApiClient() @@ -44,7 +47,7 @@ public AirbyteApiClient airbyteApiClient( .setRequestInterceptor(builder -> { builder.setHeader("User-Agent", "WorkerApp"); if (!airbyteApiAuthHeaderName.isBlank()) { - builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken); + builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken.get()); } })); } @@ -66,7 +69,7 @@ public String internalApiScheme(final WorkerPlane workerPlane) { *

* Otherwise, use the AIRBYTE_API_AUTH_HEADER_VALUE from EnvConfigs. */ - @Singleton + @Prototype @Named("internalApiAuthToken") public String internalApiAuthToken( @Value("${airbyte.internal.api.auth-header.value}") final String airbyteApiAuthHeaderValue, From fb75984425d2aab4b2c5a9580d9b86bead38f3f9 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Wed, 21 Sep 2022 12:19:23 -0500 Subject: [PATCH 2/4] add comments on why using provider and prototype: --- .../java/io/airbyte/workers/config/ApiClientBeanFactory.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java index cce6c8fa22f6..07daa08d9a85 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java @@ -49,6 +49,8 @@ public AirbyteApiClient airbyteApiClient( .setHttpClientBuilder(HttpClient.newBuilder().version(Version.HTTP_1_1)) .setRequestInterceptor(builder -> { builder.setHeader("User-Agent", "WorkerApp"); + // internalApiAuthToken is in BeanProvider because we want to create a new token each + // time we send a request. if (!airbyteApiAuthHeaderName.isBlank()) { builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken.get()); } @@ -65,7 +67,8 @@ public String internalApiScheme(final WorkerPlane workerPlane) { /** * Generate an auth token based on configs. This is called by the Api Client's requestInterceptor - * for each request. + * for each request. Using Prototype annotation here to make sure each time it's used it will + * generate a new JWT Signature if it's on data plane. *

* For Data Plane workers, generate a signed JWT as described here: * https://cloud.google.com/endpoints/docs/openapi/service-account-authentication From 4728f9b3c7382a62e48b644bee791ac7e38a7034 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Fri, 23 Sep 2022 14:13:19 -0500 Subject: [PATCH 3/4] secret hydrator fix for data plane --- .../airbyte/workers/config/ApiClientBeanFactory.java | 5 ++--- .../workers/config/SecretPersistenceBeanFactory.java | 11 ----------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java index 07daa08d9a85..5dd30b9954d5 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java @@ -10,9 +10,10 @@ import com.google.auth.oauth2.ServiceAccountCredentials; import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.config.Configs.WorkerPlane; +import io.micronaut.context.BeanProvider; import io.micronaut.context.annotation.Factory; -import io.micronaut.context.annotation.Value; import io.micronaut.context.annotation.Prototype; +import io.micronaut.context.annotation.Value; import java.io.FileInputStream; import java.net.http.HttpClient; import java.net.http.HttpClient.Version; @@ -22,8 +23,6 @@ import javax.inject.Named; import javax.inject.Singleton; import lombok.extern.slf4j.Slf4j; -import io.micronaut.context.BeanProvider; - /** * Micronaut bean factory for API client singletons. diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java index a01345ecf72f..729784699e72 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java @@ -52,8 +52,6 @@ public SecretPersistence localTestingSecretPersistence(@Named("configDatabase") @Singleton @Requires(property = "airbyte.secret.persistence", value = "GOOGLE_SECRET_MANAGER") - @Requires(property = "airbyte.worker.plane", - notEquals = "DATA_PLANE") @Named("secretPersistence") public SecretPersistence googleSecretPersistence(@Value("${airbyte.secret.store.gcp.credentials}") final String credentials, @Value("${airbyte.secret.store.gcp.project-id}") final String projectId) { @@ -73,16 +71,7 @@ public SecretPersistence vaultSecretPersistence(@Value("${airbyte.secret.store.v } @Singleton - @Requires(property = "airbyte.worker.plane", - notEquals = "DATA_PLANE") public SecretsHydrator secretsHydrator(@Named("secretPersistence") final SecretPersistence secretPersistence) { return new RealSecretsHydrator(secretPersistence); } - - @Singleton - @Requires(env = "data") - public SecretsHydrator secretsHydrator() { - return new NoOpSecretsHydrator(); - } - } From 2b54c1aec15944de6462fff9e34c5343051592fb Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Mon, 26 Sep 2022 10:23:58 -0700 Subject: [PATCH 4/4] style fix --- .../io/airbyte/workers/config/SecretPersistenceBeanFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java index eb120bee0c94..2943a5166ed9 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/SecretPersistenceBeanFactory.java @@ -6,7 +6,6 @@ import io.airbyte.config.persistence.split_secrets.GoogleSecretManagerPersistence; import io.airbyte.config.persistence.split_secrets.LocalTestingSecretPersistence; -import io.airbyte.config.persistence.split_secrets.NoOpSecretsHydrator; import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.config.persistence.split_secrets.SecretsHydrator; @@ -74,4 +73,5 @@ public SecretPersistence vaultSecretPersistence(@Value("${airbyte.secret.store.v public SecretsHydrator secretsHydrator(@Named("secretPersistence") final SecretPersistence secretPersistence) { return new RealSecretsHydrator(secretPersistence); } + }