From 350a4d3ff0de41ea92a9ac66dfe06d86448527b8 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Thu, 24 Feb 2022 09:18:42 -0800 Subject: [PATCH 1/4] Update temporal retention TTL from 7 to 30 days --- .../main/java/io/airbyte/workers/temporal/TemporalUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java index ad8e8021ae6d..c52c79cde7a0 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java @@ -71,7 +71,7 @@ public static WorkflowServiceStubs createTemporalService(final String temporalHo public static final String DEFAULT_NAMESPACE = "default"; - private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(7); + private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(30); private static final String HUMAN_READABLE_WORKFLOW_EXECUTION_TTL = DurationFormatUtils.formatDurationWords(WORKFLOW_EXECUTION_TTL.toMillis(), true, true); From 928f6bdd244004ca258ba831826457e47ca20975 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 1 Mar 2022 10:13:23 -0800 Subject: [PATCH 2/4] Move number of days to config --- .../models/src/main/java/io/airbyte/config/Configs.java | 5 +++++ .../models/src/main/java/io/airbyte/config/EnvConfigs.java | 7 +++++++ .../java/io/airbyte/workers/temporal/TemporalUtils.java | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java index d496d4edd340..33c7e1e8497a 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java @@ -178,6 +178,11 @@ public interface Configs { */ String getTemporalHost(); + /** + * Define the number of retention days for the temporal history + */ + int getTemporalRetentionInDays(); + /** * Define the url where the Airbyte Server is hosted at. Airbyte services use this information. * Manipulates the `INTERNAL_API_HOST` variable. diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java index 353b7c300177..68f0fe6ff506 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -73,6 +73,7 @@ public class EnvConfigs implements Configs { public static final String MAX_SYNC_WORKERS = "MAX_SYNC_WORKERS"; private static final String TEMPORAL_HOST = "TEMPORAL_HOST"; private static final String TEMPORAL_WORKER_PORTS = "TEMPORAL_WORKER_PORTS"; + private static final String TEMPORAL_HISTORY_RETENTION_IN_DAYS = "TEMPORAL_HISTORY_RETENTION_IN_DAYS"; public static final String JOB_KUBE_NAMESPACE = "JOB_KUBE_NAMESPACE"; private static final String SUBMITTER_NUM_THREADS = "SUBMITTER_NUM_THREADS"; public static final String JOB_MAIN_CONTAINER_CPU_REQUEST = "JOB_MAIN_CONTAINER_CPU_REQUEST"; @@ -166,6 +167,8 @@ public class EnvConfigs implements Configs { public static final String DEFAULT_NETWORK = "host"; + public static final int DEFAULT_TEMPORAL_HISTORY_RETENTION_IN_DAYS = 30; + private final Function getEnv; private final Supplier> getAllEnvKeys; private final LogConfigs logConfigs; @@ -385,6 +388,10 @@ public String getTemporalHost() { return getEnvOrDefault(TEMPORAL_HOST, "airbyte-temporal:7233"); } + @Override public int getTemporalRetentionInDays() { + return getEnvOrDefault(TEMPORAL_HISTORY_RETENTION_IN_DAYS, DEFAULT_TEMPORAL_HISTORY_RETENTION_IN_DAYS); + } + @Override public String getAirbyteApiHost() { return getEnsureEnv(INTERNAL_API_HOST).split(":")[0]; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java index c52c79cde7a0..a23405cad924 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java @@ -71,7 +71,7 @@ public static WorkflowServiceStubs createTemporalService(final String temporalHo public static final String DEFAULT_NAMESPACE = "default"; - private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(30); + private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(configs.getTemporalRetentionInDays()); private static final String HUMAN_READABLE_WORKFLOW_EXECUTION_TTL = DurationFormatUtils.formatDurationWords(WORKFLOW_EXECUTION_TTL.toMillis(), true, true); From 69e0686142b21c4b41c6ddfe7a994cb1b89d2533 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 1 Mar 2022 10:19:50 -0800 Subject: [PATCH 3/4] Format --- .../models/src/main/java/io/airbyte/config/EnvConfigs.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java index 68f0fe6ff506..5f8bd1f6e8dc 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -388,7 +388,8 @@ public String getTemporalHost() { return getEnvOrDefault(TEMPORAL_HOST, "airbyte-temporal:7233"); } - @Override public int getTemporalRetentionInDays() { + @Override + public int getTemporalRetentionInDays() { return getEnvOrDefault(TEMPORAL_HISTORY_RETENTION_IN_DAYS, DEFAULT_TEMPORAL_HISTORY_RETENTION_IN_DAYS); } From 318d805ac51039fbed40960f0232898a1da2731d Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 1 Mar 2022 11:50:12 -0800 Subject: [PATCH 4/4] Fix build --- .../models/src/main/java/io/airbyte/config/EnvConfigs.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java index 5f8bd1f6e8dc..fe80f65856ad 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -823,6 +823,10 @@ public long getEnvOrDefault(final String key, final long defaultValue) { return getEnvOrDefault(key, defaultValue, Long::parseLong, false); } + public int getEnvOrDefault(final String key, final int defaultValue) { + return getEnvOrDefault(key, defaultValue, Integer::parseInt, false); + } + public boolean getEnvOrDefault(final String key, final boolean defaultValue) { return getEnvOrDefault(key, defaultValue, Boolean::parseBoolean); }