Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update temporal retention TTL from 7 to 30 days #10635

Merged
merged 5 commits into from
Mar 1, 2022
Merged

Update temporal retention TTL from 7 to 30 days #10635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
350a4d3
Update temporal retention TTL from 7 to 30 days
benmoriceau Feb 24, 2022
31d0dc4
Merge branch 'master' of github.com:airbytehq/airbyte into bmoric/inc…
benmoriceau Mar 1, 2022
928f6bd
Move number of days to config
benmoriceau Mar 1, 2022
69e0686
Format
benmoriceau Mar 1, 2022
318d805
Fix build
benmoriceau Mar 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions airbyte-config/models/src/main/java/io/airbyte/config/Configs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,11 @@ public interface Configs {
*/
String getTemporalHost();

/**
* Define the number of retention days for the temporal history
*/
int getTemporalRetentionInDays();

/**
* Define the url where the Airbyte Server is hosted at. Airbyte services use this information.
* Manipulates the `INTERNAL_API_HOST` variable.
Expand Down
12 changes: 12 additions & 0 deletions airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ public class EnvConfigs implements Configs {
public static final String MAX_SYNC_WORKERS = "MAX_SYNC_WORKERS";
private static final String TEMPORAL_HOST = "TEMPORAL_HOST";
private static final String TEMPORAL_WORKER_PORTS = "TEMPORAL_WORKER_PORTS";
private static final String TEMPORAL_HISTORY_RETENTION_IN_DAYS = "TEMPORAL_HISTORY_RETENTION_IN_DAYS";
public static final String JOB_KUBE_NAMESPACE = "JOB_KUBE_NAMESPACE";
private static final String SUBMITTER_NUM_THREADS = "SUBMITTER_NUM_THREADS";
public static final String JOB_MAIN_CONTAINER_CPU_REQUEST = "JOB_MAIN_CONTAINER_CPU_REQUEST";
Expand Down Expand Up @@ -166,6 +167,8 @@ public class EnvConfigs implements Configs {

public static final String DEFAULT_NETWORK = "host";

public static final int DEFAULT_TEMPORAL_HISTORY_RETENTION_IN_DAYS = 30;

private final Function<String, String> getEnv;
private final Supplier<Set<String>> getAllEnvKeys;
private final LogConfigs logConfigs;
Expand Down Expand Up @@ -385,6 +388,11 @@ public String getTemporalHost() {
return getEnvOrDefault(TEMPORAL_HOST, "airbyte-temporal:7233");
}

@Override
public int getTemporalRetentionInDays() {
return getEnvOrDefault(TEMPORAL_HISTORY_RETENTION_IN_DAYS, DEFAULT_TEMPORAL_HISTORY_RETENTION_IN_DAYS);
}

@Override
public String getAirbyteApiHost() {
return getEnsureEnv(INTERNAL_API_HOST).split(":")[0];
Expand Down Expand Up @@ -815,6 +823,10 @@ public long getEnvOrDefault(final String key, final long defaultValue) {
return getEnvOrDefault(key, defaultValue, Long::parseLong, false);
}

public int getEnvOrDefault(final String key, final int defaultValue) {
return getEnvOrDefault(key, defaultValue, Integer::parseInt, false);
}

public boolean getEnvOrDefault(final String key, final boolean defaultValue) {
return getEnvOrDefault(key, defaultValue, Boolean::parseBoolean);
}
Expand Down
2 changes: 1 addition & 1 deletion airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public static WorkflowServiceStubs createTemporalService(final String temporalHo

public static final String DEFAULT_NAMESPACE = "default";

private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(7);
private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(configs.getTemporalRetentionInDays());
private static final String HUMAN_READABLE_WORKFLOW_EXECUTION_TTL =
DurationFormatUtils.formatDurationWords(WORKFLOW_EXECUTION_TTL.toMillis(), true, true);

Expand Down