Skip to content

Commit

Permalink
Export PageStorage V3 config into global config (#4502)
Browse files Browse the repository at this point in the history 
ref #3594
  • Loading branch information
@jiaqizho
jiaqizho authored Apr 29, 2022
1 parent 4019600 commit 63cae97
Show file tree
Hide file tree
Showing 19 changed files with 191 additions and 57 deletions.
8 changes: 6 additions & 2 deletions dbms/src/Interpreters/Settings.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,8 @@ struct Settings
M(SettingUInt64, dt_segment_delta_small_column_file_size, 8388608, "Determine whether a column file in delta is small or not. 8MB by default.") \
M(SettingUInt64, dt_segment_stable_pack_rows, DEFAULT_MERGE_BLOCK_SIZE, "Expected stable pack rows in DeltaTree Engine.") \
M(SettingFloat, dt_segment_wait_duration_factor, 1, "The factor of wait duration in a write stall.") \
M(SettingUInt64, dt_bg_gc_check_interval, 5, "Background gc thread check interval, the unit is second.") \
M(SettingInt64, dt_bg_gc_max_segments_to_check_every_round, 100, "Max segments to check in every gc round, value less than or equal to 0 means gc no segments.") \
M(SettingUInt64, dt_bg_gc_check_interval, 5, "Background gc thread check interval, the unit is second.") \
M(SettingInt64, dt_bg_gc_max_segments_to_check_every_round, 100, "Max segments to check in every gc round, value less than or equal to 0 means gc no segments.") \
M(SettingFloat, dt_bg_gc_ratio_threhold_to_trigger_gc, 1.2, "Trigger segment's gc when the ratio of invalid version exceed this threhold. Values smaller than or equal to 1.0 means gc all " \
"segments") \
M(SettingFloat, dt_bg_gc_delta_delete_ratio_to_trigger_gc, 0.3, "Trigger segment's gc when the ratio of delta delete range to stable exceeds this ratio.") \
Expand All @@ -294,6 +294,7 @@ struct Settings
"`dt_stroage_num_max_expect_legacy_files`") \
M(SettingFloat, dt_page_gc_low_write_prob, 0.10, "Probability to run gc when write there is few writes.") \
\
\
M(SettingUInt64, dt_storage_pool_log_write_slots, 4, "Max write concurrency for each StoragePool.log.") \
M(SettingUInt64, dt_storage_pool_log_gc_min_file_num, 10, "Min number of page files to compact") \
M(SettingUInt64, dt_storage_pool_log_gc_min_legacy_num, 3, "Min number of legacy page files to compact") \
Expand All @@ -313,6 +314,9 @@ struct Settings
M(SettingFloat, dt_storage_pool_meta_gc_max_valid_rate, 0.35, "Max valid rate of deciding a page file can be compact") \
\
M(SettingUInt64, dt_checksum_frame_size, DBMS_DEFAULT_BUFFER_SIZE, "Frame size for delta tree stable storage") \
\
M(SettingDouble, dt_storage_blob_heavy_gc_valid_rate, 0.2, "Max valid rate of deciding a blob can be compact") \
\
M(SettingChecksumAlgorithm, dt_checksum_algorithm, ChecksumAlgo::XXH3, "Checksum algorithm for delta tree stable storage") \
M(SettingCompressionMethod, dt_compression_method, CompressionMethod::LZ4, "The method of data compression when writing.") \
M(SettingInt64, dt_compression_level, 1, "The compression level.") \
Expand Down
12 changes: 11 additions & 1 deletion dbms/src/Server/tests/gtest_server_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ max_rows_in_set = 455
dt_segment_limit_rows = 1000005
dt_enable_rough_set_filter = 0
max_memory_usage = 102000
dt_storage_blob_heavy_gc_valid_rate = 0.2
dt_storage_pool_data_gc_min_file_num = 8
dt_storage_pool_data_gc_min_legacy_num = 2
dt_storage_pool_data_gc_min_bytes = 256
Expand Down Expand Up @@ -221,6 +222,7 @@ dt_compression_level = 1
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_min_file_num, 8);
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_min_legacy_num, 2);
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_min_bytes, 256);
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_blob_heavy_gc_valid_rate, 0.2);
ASSERT_EQ(global_ctx.getSettingsRef().dt_segment_delta_small_column_file_size, 8388608);
ASSERT_EQ(global_ctx.getSettingsRef().dt_segment_delta_small_column_file_rows, 2048);
ASSERT_EQ(global_ctx.getSettingsRef().dt_segment_limit_size, 536870912);
Expand Down Expand Up @@ -274,6 +276,7 @@ try
max_rows_in_set = 455
dt_segment_limit_rows = 1000005
dt_enable_rough_set_filter = 0
dt_storage_blob_heavy_gc_valid_rate = 0.3
max_memory_usage = 102000
dt_storage_pool_data_gc_min_file_num = 8
dt_storage_pool_data_gc_min_legacy_num = 2
Expand All @@ -295,16 +298,18 @@ dt_page_gc_low_write_prob = 0.2
EXPECT_NE(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num);
EXPECT_NE(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes);
EXPECT_NE(cfg.gc_max_valid_rate, settings.dt_storage_pool_data_gc_max_valid_rate);
EXPECT_NE(cfg.blob_heavy_gc_valid_rate, settings.dt_storage_blob_heavy_gc_valid_rate);
EXPECT_NE(cfg.open_file_max_idle_time, settings.dt_open_file_max_idle_seconds);
EXPECT_NE(cfg.prob_do_gc_when_write_is_low, settings.dt_page_gc_low_write_prob * 1000);

persister.gc();

cfg = persister.page_storage->getSettings();

EXPECT_NE(cfg.gc_min_files, settings.dt_storage_pool_data_gc_min_file_num);
EXPECT_NE(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num);
EXPECT_NE(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes);
EXPECT_NE(cfg.gc_max_valid_rate, settings.dt_storage_pool_data_gc_max_valid_rate);
EXPECT_EQ(cfg.blob_heavy_gc_valid_rate, settings.dt_storage_blob_heavy_gc_valid_rate);
EXPECT_EQ(cfg.open_file_max_idle_time, settings.dt_open_file_max_idle_seconds);
EXPECT_EQ(cfg.prob_do_gc_when_write_is_low, settings.dt_page_gc_low_write_prob * 1000);
};
Expand All @@ -327,6 +332,7 @@ dt_page_gc_low_write_prob = 0.2
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_min_legacy_num, 2);
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_min_bytes, 256);
ASSERT_FLOAT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_max_valid_rate, 0.5);
ASSERT_DOUBLE_EQ(global_ctx.getSettingsRef().dt_storage_blob_heavy_gc_valid_rate, 0.3);
ASSERT_EQ(global_ctx.getSettingsRef().dt_open_file_max_idle_seconds, 20);
ASSERT_FLOAT_EQ(global_ctx.getSettingsRef().dt_page_gc_low_write_prob, 0.2);
verify_persister_reload_config(persister);
Expand All @@ -346,6 +352,7 @@ max_rows_in_set = 455
dt_segment_limit_rows = 1000005
dt_enable_rough_set_filter = 0
max_memory_usage = 102000
dt_storage_blob_heavy_gc_valid_rate = 0.3
dt_storage_pool_data_gc_min_file_num = 8
dt_storage_pool_data_gc_min_legacy_num = 2
dt_storage_pool_data_gc_min_bytes = 256
Expand All @@ -366,6 +373,7 @@ dt_page_gc_low_write_prob = 0.2
EXPECT_NE(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num);
EXPECT_NE(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes);
EXPECT_NE(cfg.gc_max_valid_rate, settings.dt_storage_pool_data_gc_max_valid_rate);
EXPECT_NE(cfg.blob_heavy_gc_valid_rate, settings.dt_storage_blob_heavy_gc_valid_rate);
EXPECT_NE(cfg.open_file_max_idle_time, settings.dt_open_file_max_idle_seconds);
EXPECT_NE(cfg.prob_do_gc_when_write_is_low, settings.dt_page_gc_low_write_prob * 1000);

Expand All @@ -376,6 +384,7 @@ dt_page_gc_low_write_prob = 0.2
EXPECT_EQ(cfg.gc_min_legacy_num, settings.dt_storage_pool_data_gc_min_legacy_num);
EXPECT_EQ(cfg.gc_min_bytes, settings.dt_storage_pool_data_gc_min_bytes);
EXPECT_DOUBLE_EQ(cfg.gc_max_valid_rate, settings.dt_storage_pool_data_gc_max_valid_rate);
EXPECT_DOUBLE_EQ(cfg.blob_heavy_gc_valid_rate, settings.dt_storage_blob_heavy_gc_valid_rate);
EXPECT_EQ(cfg.open_file_max_idle_time, settings.dt_open_file_max_idle_seconds);
EXPECT_EQ(cfg.prob_do_gc_when_write_is_low, settings.dt_page_gc_low_write_prob * 1000);
};
Expand All @@ -398,6 +407,7 @@ dt_page_gc_low_write_prob = 0.2
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_min_legacy_num, 2);
ASSERT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_min_bytes, 256);
ASSERT_FLOAT_EQ(global_ctx.getSettingsRef().dt_storage_pool_data_gc_max_valid_rate, 0.5);
ASSERT_DOUBLE_EQ(global_ctx.getSettingsRef().dt_storage_blob_heavy_gc_valid_rate, 0.3);
ASSERT_EQ(global_ctx.getSettingsRef().dt_open_file_max_idle_seconds, 20);
ASSERT_FLOAT_EQ(global_ctx.getSettingsRef().dt_page_gc_low_write_prob, 0.2);
verify_storage_pool_reload_config(storage_pool);
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Storages/DeltaMerge/StoragePool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ PageStorage::Config extractConfig(const Settings & settings, StorageType subtype
config.gc_min_files = settings.dt_storage_pool_##NAME##_gc_min_file_num; \
config.gc_min_bytes = settings.dt_storage_pool_##NAME##_gc_min_bytes; \
config.gc_min_legacy_num = settings.dt_storage_pool_##NAME##_gc_min_legacy_num; \
config.gc_max_valid_rate = settings.dt_storage_pool_##NAME##_gc_max_valid_rate;
config.gc_max_valid_rate = settings.dt_storage_pool_##NAME##_gc_max_valid_rate; \
config.blob_heavy_gc_valid_rate = settings.dt_storage_blob_heavy_gc_valid_rate;

PageStorage::Config config = getConfigFromSettings(settings);


switch (subtype)
{
case StorageType::Log:
Expand Down
3 changes: 3 additions & 0 deletions dbms/src/Storages/Page/ConfigSettings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ void mergeConfigFromSettings(const DB::Settings & settings, PageStorage::Config
config.gc_max_expect_legacy_files = settings.dt_page_num_max_expect_legacy_files;
if (settings.dt_page_num_max_gc_valid_rate > 0.0)
config.gc_max_valid_rate_bound = settings.dt_page_num_max_gc_valid_rate;

// V3 setting which export to global setting
config.blob_heavy_gc_valid_rate = settings.dt_storage_blob_heavy_gc_valid_rate;
}

PageStorage::Config getConfigFromSettings(const DB::Settings & settings)
Expand Down
18 changes: 9 additions & 9 deletions dbms/src/Storages/Page/PageDefines.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,21 @@ using Seconds = std::chrono::seconds;
static constexpr UInt64 MB = 1ULL * 1024 * 1024;
static constexpr UInt64 GB = MB * 1024;


// PageStorage V2 define
static constexpr UInt64 PAGE_SIZE_STEP = (1 << 10) * 16; // 16 KB
static constexpr UInt64 PAGE_BUFFER_SIZE = DBMS_DEFAULT_BUFFER_SIZE;
static constexpr UInt64 PAGE_MAX_BUFFER_SIZE = 128 * MB;
static constexpr UInt64 PAGE_SPLIT_SIZE = 1 * MB;
static constexpr UInt64 PAGE_FILE_MAX_SIZE = 1024 * 2 * MB;
static constexpr UInt64 PAGE_FILE_SMALL_SIZE = 2 * MB;
static constexpr UInt64 PAGE_FILE_ROLL_SIZE = 128 * MB;
static constexpr UInt64 PAGE_META_ROLL_SIZE = 2 * MB;

static constexpr UInt64 BLOBFILE_LIMIT_SIZE = 512 * MB;
static constexpr UInt64 BLOBSTORE_CACHED_FD_SIZE = 100;

static_assert(PAGE_SIZE_STEP >= ((1 << 10) * 16), "PAGE_SIZE_STEP should be at least 16 KB");
static_assert((PAGE_SIZE_STEP & (PAGE_SIZE_STEP - 1)) == 0, "PAGE_SIZE_STEP should be power of 2");
static_assert(PAGE_BUFFER_SIZE % PAGE_SIZE_STEP == 0, "PAGE_BUFFER_SIZE should be dividable by PAGE_SIZE_STEP");

// PageStorage V3 define
static constexpr UInt64 BLOBFILE_LIMIT_SIZE = 512 * MB;
static constexpr UInt64 BLOBSTORE_CACHED_FD_SIZE = 100;
static constexpr UInt64 PAGE_META_ROLL_SIZE = 2 * MB;
static constexpr UInt64 MAX_PERSISTED_LOG_FILES = 4;

using NamespaceId = UInt64;
static constexpr NamespaceId MAX_NAMESPACE_ID = UINT64_MAX;
Expand Down Expand Up @@ -111,7 +111,7 @@ inline size_t alignPage(size_t n)
template <>
struct fmt::formatter<DB::PageIdV3Internal>
{
constexpr auto parse(format_parse_context & ctx) -> decltype(ctx.begin())
static constexpr auto parse(format_parse_context & ctx) -> decltype(ctx.begin())
{
return ctx.begin();
}
Expand Down
43 changes: 41 additions & 2 deletions dbms/src/Storages/Page/PageStorage.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ class PageStorage : private boost::noncopyable

struct Config
{
//==========================================================================================
// V2 config
//==========================================================================================
SettingBool sync_on_write = true;

SettingUInt64 file_roll_size = PAGE_FILE_ROLL_SIZE;
Expand Down Expand Up @@ -109,6 +112,18 @@ class PageStorage : private boost::noncopyable

MVCC::VersionSetConfig version_set_config;

//==========================================================================================
// V3 config
//==========================================================================================
SettingUInt64 blob_file_limit_size = BLOBFILE_LIMIT_SIZE;
SettingUInt64 blob_spacemap_type = 2;
SettingUInt64 blob_cached_fd_size = BLOBSTORE_CACHED_FD_SIZE;
SettingDouble blob_heavy_gc_valid_rate = 0.2;

SettingUInt64 wal_roll_size = PAGE_META_ROLL_SIZE;
SettingUInt64 wal_recover_mode = 0;
SettingUInt64 wal_max_persisted_log_files = MAX_PERSISTED_LOG_FILES;

void reload(const Config & rhs)
{
// Reload is not atomic, but should be good enough
Expand All @@ -122,9 +137,18 @@ class PageStorage : private boost::noncopyable
prob_do_gc_when_write_is_low = rhs.prob_do_gc_when_write_is_low;
// Reload fd idle time
open_file_max_idle_time = rhs.open_file_max_idle_time;

// Reload V3 setting
blob_file_limit_size = rhs.blob_file_limit_size;
blob_spacemap_type = rhs.blob_spacemap_type;
blob_cached_fd_size = rhs.blob_cached_fd_size;
blob_heavy_gc_valid_rate = rhs.blob_heavy_gc_valid_rate;
wal_roll_size = rhs.wal_roll_size;
wal_recover_mode = rhs.wal_recover_mode;
wal_max_persisted_log_files = rhs.wal_max_persisted_log_files;
}

String toDebugString() const
String toDebugStringV2() const
{
return fmt::format(
"PageStorage::Config {{gc_min_files: {}, gc_min_bytes:{}, gc_force_hardlink_rate: {:.3f}, gc_max_valid_rate: {:.3f}, "
Expand All @@ -140,11 +164,26 @@ class PageStorage : private boost::noncopyable
prob_do_gc_when_write_is_low,
open_file_max_idle_time);
}

String toDebugStringV3() const
{
return fmt::format(
"PageStorage::Config V3 {{"
"blob_file_limit_size: {}, blob_spacemap_type: {}, "
"blob_cached_fd_size: {}, blob_heavy_gc_valid_rate: {:.3f}, "
"wal_roll_size: {}, wal_recover_mode: {}, wal_max_persisted_log_files: {}}}",
blob_file_limit_size.get(),
blob_spacemap_type.get(),
blob_cached_fd_size.get(),
blob_heavy_gc_valid_rate.get(),
wal_roll_size.get(),
wal_recover_mode.get(),
wal_max_persisted_log_files.get());
}
};
void reloadSettings(const Config & new_config) { config.reload(new_config); };
Config getSettings() const { return config; }


public:
static PageStoragePtr
create(
Expand Down
16 changes: 14 additions & 2 deletions dbms/src/Storages/Page/V2/gc/DataCompactor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,13 @@ DataCompactor<SnapshotPtr>::tryMigrate( //
}
else
{
LOG_FMT_DEBUG(log, "{} DataCompactor::tryMigrate exit without compaction [candidates size={}] [total byte size={}], [files without valid page={}] Config{{ {} }}", storage_name, result.candidate_size, result.bytes_migrate, candidates.files_without_valid_pages.size(), config.toDebugString());
LOG_FMT_DEBUG(log, "{} DataCompactor::tryMigrate exit without compaction [candidates size={}] [total byte size={}], "
"[files without valid page={}] Config{{ {} }}", //
storage_name,
result.candidate_size,
result.bytes_migrate,
candidates.files_without_valid_pages.size(),
config.toDebugStringV2());
}

return {result, std::move(migrate_entries_edit)};
Expand Down Expand Up @@ -555,7 +561,13 @@ void DataCompactor<SnapshotPtr>::logMigrationDetails(const MigrateInfos & infos,
}
migrate_stream << "]";
remove_stream << "]";
LOG_FMT_DEBUG(log, "{} Migrate pages to PageFile_{}_{}, migrate: {}, remove: {}, Config{{ {} }}", storage_name, migrate_file_id.first, migrate_file_id.second, migrate_stream.str(), remove_stream.str(), config.toDebugString());
LOG_FMT_DEBUG(log, "{} Migrate pages to PageFile_{}_{}, migrate: {}, remove: {}, Config{{ {} }}", //
storage_name,
migrate_file_id.first,
migrate_file_id.second,
migrate_stream.str(),
remove_stream.str(),
config.toDebugStringV2());
}


Expand Down
Loading

0 comments on commit 63cae97

Please sign in to comment.