Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Configurable network error retry policy #1782

Merged
merged 3 commits into from
Aug 5, 2024
Merged

Configurable network error retry policy #1782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3fd4c92
Configurable network error retry policy
muhamadazmy Aug 2, 2024
4b7994a
set default to exponential delay
muhamadazmy Aug 5, 2024
babb3c1
the Default implementation is used anyway
muhamadazmy Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 6 additions & 11 deletions crates/node/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ impl Node {
metadata_store_client: &MetadataStoreClient,
config: &Configuration,
) -> Result<FixedPartitionTable, Error> {
Self::retry_on_network_error(|| {
Self::retry_on_network_error(config.common.network_error_retry_policy.clone(), || {
metadata_store_client.get_or_insert(PARTITION_TABLE_KEY.clone(), || {
FixedPartitionTable::new(Version::MIN, config.common.bootstrap_num_partitions())
})
Expand All @@ -459,7 +459,7 @@ impl Node {
config: &Configuration,
num_partitions: u64,
) -> Result<Logs, Error> {
Self::retry_on_network_error(|| {
Self::retry_on_network_error(config.common.network_error_retry_policy.clone(), || {
metadata_store_client.get_or_insert(BIFROST_CONFIG_KEY.clone(), || {
bootstrap_logs_metadata(config.bifrost.default_provider, num_partitions)
})
Expand All @@ -472,7 +472,7 @@ impl Node {
metadata_store_client: &MetadataStoreClient,
common_opts: &CommonOptions,
) -> Result<NodesConfiguration, Error> {
Self::retry_on_network_error(|| {
Self::retry_on_network_error(common_opts.network_error_retry_policy.clone(), || {
let mut previous_node_generation = None;
metadata_store_client.read_modify_write(NODES_CONFIG_KEY.clone(), move |nodes_config| {
let mut nodes_config = if common_opts.allow_bootstrap {
Expand Down Expand Up @@ -553,22 +553,17 @@ impl Node {
.map_err(|err| err.transpose())
}

async fn retry_on_network_error<Fn, Fut, T, E>(action: Fn) -> Result<T, E>
async fn retry_on_network_error<Fn, Fut, T, E, P>(retry_policy: P, action: Fn) -> Result<T, E>
where
P: Into<RetryPolicy>,
Fn: FnMut() -> Fut,
Fut: Future<Output = Result<T, E>>,
E: MetadataStoreClientError + std::fmt::Display,
{
// todo: Make upsert timeout configurable
let retry_policy = RetryPolicy::exponential(
Duration::from_millis(10),
2.0,
Some(15),
Some(Duration::from_secs(5)),
);
let upsert_start = Instant::now();

retry_policy
.into()
.retry_if(action, |err: &E| {
if err.is_network_error() {
if upsert_start.elapsed() < Duration::from_secs(5) {
Expand Down
11 changes: 11 additions & 0 deletions crates/types/src/config/common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,11 @@ pub struct CommonOptions {
#[serde(with = "serde_with::As::<serde_with::DisplayFromStr>")]
#[cfg_attr(feature = "schemars", schemars(with = "String"))]
pub metadata_update_interval: humantime::Duration,

/// # Network error retry policy
///
/// The retry policy for node network error
pub network_error_retry_policy: RetryPolicy,
}

static HOSTNAME: Lazy<String> = Lazy::new(|| {
Expand Down Expand Up @@ -357,6 +362,12 @@ impl Default for CommonOptions {
rocksdb_perf_level: PerfStatsLevel::EnableCount,
rocksdb: Default::default(),
metadata_update_interval: std::time::Duration::from_secs(3).into(),
network_error_retry_policy: RetryPolicy::exponential(
Duration::from_millis(10),
2.0,
Some(15),
Some(Duration::from_secs(5)),
),
}
}
}
Expand Down
Loading