Skip to content

Commit

Permalink
fix: support to compat the old layered memtable options (#1568)
Browse files Browse the repository at this point in the history
## Rationale
We introduce the explicit flag to control should we enable layered
memtable, but it has some compatibility problem when upgrading from old
version.
This pr add an option to support compating the old layered memtable
on/off control method.

## Detailed Changes
Add an option to support compating the old layered memtable on/off
control method.

## Test Plan
Manually.
  • Loading branch information
Rachelint authored Sep 14, 2024
1 parent 645a8b3 commit 3bcc64f
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 9 deletions.
1 change: 1 addition & 0 deletions src/analytic_engine/src/instance/open.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ impl Instance {
preflush_write_buffer_size_ratio: ctx.config.preflush_write_buffer_size_ratio,
manifest_snapshot_every_n_updates: ctx.config.manifest.snapshot_every_n_updates,
enable_primary_key_sampling: ctx.config.enable_primary_key_sampling,
try_compat_old_layered_memtable_opts: ctx.config.try_compat_old_layered_memtable_opts,
metrics_opt: ctx.config.metrics.clone(),
});
let manifest = ManifestImpl::open(
Expand Down
7 changes: 7 additions & 0 deletions src/analytic_engine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ pub struct Config {
/// Default options for table
pub table_opts: TableOptions,

/// Should we try to compat the `LayeredMemtableOptions` in `TableOptions`
/// The old one use if `mutable_segment_switch_threshold` > 0 to control
/// the on/off of layered memtable(`0`:off, `>0`:on).
/// The new one use a explicit flag `enable` to do that.
pub try_compat_old_layered_memtable_opts: bool,

pub compaction: SchedulerConfig,

/// sst meta cache capacity
Expand Down Expand Up @@ -179,6 +185,7 @@ impl Default for Config {
replay_batch_size: 500,
max_replay_tables_per_batch: 64,
table_opts: TableOptions::default(),
try_compat_old_layered_memtable_opts: false,
compaction: SchedulerConfig::default(),
sst_meta_cache_cap: Some(1000),
sst_data_cache_cap: Some(1000),
Expand Down
1 change: 1 addition & 0 deletions src/analytic_engine/src/manifest/details.rs
Original file line number Diff line number Diff line change
Expand Up @@ -776,6 +776,7 @@ mod tests {
manifest_snapshot_every_n_updates: NonZeroUsize::new(usize::MAX).unwrap(),
metrics_opt: MetricsOptions::default(),
enable_primary_key_sampling: false,
try_compat_old_layered_memtable_opts: false,
},
&purger,
mem_size_options,
Expand Down
38 changes: 29 additions & 9 deletions src/analytic_engine/src/table/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ use crate::{
sst_util,
version::{MemTableForWrite, MemTableState, SamplingMemTable, TableVersion},
},
table_options::UpdateMode,
MetricsOptions, TableOptions,
};

Expand Down Expand Up @@ -155,6 +156,7 @@ pub struct TableConfig {
pub manifest_snapshot_every_n_updates: NonZeroUsize,
pub metrics_opt: MetricsOptions,
pub enable_primary_key_sampling: bool,
pub try_compat_old_layered_memtable_opts: bool,
}

#[derive(Debug, Clone)]
Expand Down Expand Up @@ -314,11 +316,13 @@ impl TableData {
name,
schema,
} = desc;

let TableConfig {
preflush_write_buffer_size_ratio,
manifest_snapshot_every_n_updates,
metrics_opt,
enable_primary_key_sampling,
..
} = config;

let memtable_factory: MemTableFactoryRef = match opts.memtable_type {
Expand Down Expand Up @@ -406,6 +410,7 @@ impl TableData {
manifest_snapshot_every_n_updates,
metrics_opt,
enable_primary_key_sampling,
try_compat_old_layered_memtable_opts,
} = config;

let memtable_factory: MemTableFactoryRef = match add_meta.opts.memtable_type {
Expand All @@ -421,17 +426,31 @@ impl TableData {
.mutable_segment_switch_threshold
.0 as usize;

ensure!(
mutable_segment_switch_threshold > 0,
InvalidTableOpts {
if mutable_segment_switch_threshold > 0 {
ensure!(
add_meta.opts.update_mode != UpdateMode::Overwrite,
InvalidTableOpts {
msg: "layered memtable is enabled but update mode is Overwrite",
}
);

Arc::new(LayeredMemtableFactory::new(
memtable_factory,
mutable_segment_switch_threshold,
)) as _
} else if try_compat_old_layered_memtable_opts {
// Maybe some old layered memtable opts controlling the on/off of this feature
// by checking `mutable_segment_switch_threshold`(`0`:disable, `>0`:enable)
// were persisted.
// If `try_compat_old_layered_memtable_opts` is true, we will try to follow the
// old behavior.
memtable_factory as _
} else {
return InvalidTableOpts {
msg: "layered memtable is enabled but mutable_switch_threshold is 0",
}
);

Arc::new(LayeredMemtableFactory::new(
memtable_factory,
mutable_segment_switch_threshold,
)) as _
.fail();
}
} else {
memtable_factory as _
};
Expand Down Expand Up @@ -1028,6 +1047,7 @@ pub mod tests {
manifest_snapshot_every_n_updates: self.manifest_snapshot_every_n_updates,
metrics_opt: MetricsOptions::default(),
enable_primary_key_sampling: false,
try_compat_old_layered_memtable_opts: false,
},
&purger,
mem_size_options,
Expand Down
4 changes: 4 additions & 0 deletions src/analytic_engine/src/table_meta_set_impl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ pub(crate) struct TableMetaSetImpl {
pub(crate) preflush_write_buffer_size_ratio: f32,
pub(crate) manifest_snapshot_every_n_updates: NonZeroUsize,
pub(crate) enable_primary_key_sampling: bool,
pub(crate) try_compat_old_layered_memtable_opts: bool,
pub(crate) metrics_opt: MetricsOptions,
}

Expand Down Expand Up @@ -140,6 +141,8 @@ impl TableMetaSetImpl {
.manifest_snapshot_every_n_updates,
metrics_opt: self.metrics_opt.clone(),
enable_primary_key_sampling: self.enable_primary_key_sampling,
try_compat_old_layered_memtable_opts: self
.try_compat_old_layered_memtable_opts,
},
&self.file_purger,
mem_size_options,
Expand Down Expand Up @@ -271,6 +274,7 @@ impl TableMetaSetImpl {
manifest_snapshot_every_n_updates: self.manifest_snapshot_every_n_updates,
metrics_opt: self.metrics_opt.clone(),
enable_primary_key_sampling: self.enable_primary_key_sampling,
try_compat_old_layered_memtable_opts: self.try_compat_old_layered_memtable_opts,
},
mem_size_options,
allocator,
Expand Down

0 comments on commit 3bcc64f

Please sign in to comment.