Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Remove prover db from house keeper #2795

Merged
merged 4 commits into from
Sep 4, 2024
Merged

feat: Remove prover db from house keeper #2795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
42e085d
feat: Remove prover db from house keeper
EmilLuta Sep 3, 2024
be76766
Update etc/env files
EmilLuta Sep 3, 2024
51e06e3
Remove prover dal from core
EmilLuta Sep 4, 2024
18ef3d9
Remove unnecessary code
EmilLuta Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 0 additions & 3 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,6 @@ members = [
"core/tests/test_account",
"core/tests/loadnext",
"core/tests/vm-benchmark",
# Parts of prover workspace that are needed for Core workspace
"prover/crates/lib/prover_dal",
]
resolver = "2"

Expand Down Expand Up @@ -233,7 +231,6 @@ zksync_protobuf_build = "=0.1.0-rc.11"

# "Local" dependencies
zksync_multivm = { version = "0.1.0", path = "core/lib/multivm" }
zksync_prover_dal = { version = "0.1.0", path = "prover/crates/lib/prover_dal" }
zksync_vlog = { version = "0.1.0", path = "core/lib/vlog" }
zksync_vm_interface = { version = "0.1.0", path = "core/lib/vm_interface" }
zksync_vm_executor = { version = "0.1.0", path = "core/lib/vm_executor" }
Expand Down
16 changes: 3 additions & 13 deletions core/bin/zksync_server/src/node_builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,6 @@ impl MainNodeBuilder {
let pools_layer = PoolsLayerBuilder::empty(config, secrets)
.with_master(true)
.with_replica(true)
.with_prover(true) // Used by house keeper.
EmilLuta marked this conversation as resolved.
Show resolved Hide resolved
.build();
self.node.add_layer(pools_layer);
Ok(self)
Expand Down Expand Up @@ -446,18 +445,9 @@ impl MainNodeBuilder {

fn add_house_keeper_layer(mut self) -> anyhow::Result<Self> {
let house_keeper_config = try_load_config!(self.configs.house_keeper_config);
let fri_prover_config = try_load_config!(self.configs.prover_config);
let fri_witness_generator_config = try_load_config!(self.configs.witness_generator_config);
let fri_prover_group_config = try_load_config!(self.configs.prover_group_config);
let fri_proof_compressor_config = try_load_config!(self.configs.proof_compressor_config);

self.node.add_layer(HouseKeeperLayer::new(
house_keeper_config,
fri_prover_config,
fri_witness_generator_config,
fri_prover_group_config,
fri_proof_compressor_config,
));

self.node
.add_layer(HouseKeeperLayer::new(house_keeper_config));

Ok(self)
}
Expand Down
25 changes: 0 additions & 25 deletions core/lib/config/src/configs/house_keeper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,4 @@ use serde::Deserialize;
#[derive(Debug, Deserialize, Clone, PartialEq)]
pub struct HouseKeeperConfig {
pub l1_batch_metrics_reporting_interval_ms: u64,
pub gpu_prover_queue_reporting_interval_ms: u64,
pub prover_job_retrying_interval_ms: u64,
pub prover_stats_reporting_interval_ms: u64,
pub witness_job_moving_interval_ms: u64,
pub witness_generator_stats_reporting_interval_ms: u64,
pub witness_generator_job_retrying_interval_ms: u64,
pub prover_db_pool_size: u32,
pub proof_compressor_job_retrying_interval_ms: u64,
pub proof_compressor_stats_reporting_interval_ms: u64,
pub prover_job_archiver_archiving_interval_ms: Option<u64>,
pub prover_job_archiver_archive_after_secs: Option<u64>,
pub fri_gpu_prover_archiver_archiving_interval_ms: Option<u64>,
pub fri_gpu_prover_archiver_archive_after_secs: Option<u64>,
}

impl HouseKeeperConfig {
pub fn prover_job_archiver_params(&self) -> Option<(u64, u64)> {
self.prover_job_archiver_archiving_interval_ms
.zip(self.prover_job_archiver_archive_after_secs)
}

pub fn fri_gpu_prover_archiver_params(&self) -> Option<(u64, u64)> {
self.fri_gpu_prover_archiver_archiving_interval_ms
.zip(self.fri_gpu_prover_archiver_archive_after_secs)
}
}
13 changes: 0 additions & 13 deletions core/lib/config/src/testonly.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,19 +630,6 @@ impl Distribution<configs::house_keeper::HouseKeeperConfig> for EncodeDist {
fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::house_keeper::HouseKeeperConfig {
configs::house_keeper::HouseKeeperConfig {
l1_batch_metrics_reporting_interval_ms: self.sample(rng),
gpu_prover_queue_reporting_interval_ms: self.sample(rng),
prover_job_retrying_interval_ms: self.sample(rng),
prover_stats_reporting_interval_ms: self.sample(rng),
witness_job_moving_interval_ms: self.sample(rng),
witness_generator_stats_reporting_interval_ms: self.sample(rng),
prover_db_pool_size: self.sample(rng),
witness_generator_job_retrying_interval_ms: self.sample(rng),
proof_compressor_job_retrying_interval_ms: self.sample(rng),
proof_compressor_stats_reporting_interval_ms: self.sample(rng),
prover_job_archiver_archiving_interval_ms: self.sample(rng),
prover_job_archiver_archive_after_secs: self.sample(rng),
fri_gpu_prover_archiver_archiving_interval_ms: self.sample(rng),
fri_gpu_prover_archiver_archive_after_secs: self.sample(rng),
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion core/lib/dal/src/models/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ pub mod storage_sync;
pub mod storage_tee_proof;
pub mod storage_transaction;
pub mod storage_verification_request;
pub mod storage_witness_job_info;

#[cfg(test)]
mod tests;

Expand Down
78 changes: 0 additions & 78 deletions core/lib/dal/src/models/storage_witness_job_info.rs
View file
Open in desktop

This file was deleted.

31 changes: 0 additions & 31 deletions core/lib/env_config/src/house_keeper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,6 @@ mod tests {
fn expected_config() -> HouseKeeperConfig {
HouseKeeperConfig {
l1_batch_metrics_reporting_interval_ms: 10_000,
gpu_prover_queue_reporting_interval_ms: 10_000,
prover_job_retrying_interval_ms: 10000,
prover_stats_reporting_interval_ms: 5_000,
witness_job_moving_interval_ms: 30_000,
witness_generator_stats_reporting_interval_ms: 10_000,
witness_generator_job_retrying_interval_ms: 30_000,
prover_db_pool_size: 2,
proof_compressor_job_retrying_interval_ms: 30_000,
proof_compressor_stats_reporting_interval_ms: 10_000,
prover_job_archiver_archiving_interval_ms: Some(1_800_000),
prover_job_archiver_archive_after_secs: Some(172_800),
// 24 hours
fri_gpu_prover_archiver_archiving_interval_ms: Some(86_400_000),
// 48 hours
fri_gpu_prover_archiver_archive_after_secs: Some(172_800),
}
}

Expand All @@ -41,22 +26,6 @@ mod tests {
let mut lock = MUTEX.lock();
let config = r#"
HOUSE_KEEPER_L1_BATCH_METRICS_REPORTING_INTERVAL_MS="10000"
HOUSE_KEEPER_GPU_PROVER_QUEUE_REPORTING_INTERVAL_MS="10000"
HOUSE_KEEPER_PROVER_JOB_RETRYING_INTERVAL_MS="10000"
HOUSE_KEEPER_WITNESS_JOB_MOVING_INTERVAL_MS="30000"
HOUSE_KEEPER_WITNESS_GENERATOR_STATS_REPORTING_INTERVAL_MS="10000"
HOUSE_KEEPER_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000"
HOUSE_KEEPER_FRI_WITNESS_JOB_MOVING_INTERVAL_MS="40000"
HOUSE_KEEPER_FRI_PROVER_JOB_RETRYING_INTERVAL_MS="30000"
HOUSE_KEEPER_FRI_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000"
HOUSE_KEEPER_PROVER_DB_POOL_SIZE="2"
HOUSE_KEEPER_PROVER_STATS_REPORTING_INTERVAL_MS="5000"
HOUSE_KEEPER_PROOF_COMPRESSOR_STATS_REPORTING_INTERVAL_MS="10000"
HOUSE_KEEPER_PROOF_COMPRESSOR_JOB_RETRYING_INTERVAL_MS="30000"
HOUSE_KEEPER_PROVER_JOB_ARCHIVER_ARCHIVING_INTERVAL_MS="1800000"
HOUSE_KEEPER_PROVER_JOB_ARCHIVER_ARCHIVE_AFTER_SECS="172800"
HOUSE_KEEPER_FRI_GPU_PROVER_ARCHIVER_ARCHIVING_INTERVAL_MS="86400000"
HOUSE_KEEPER_FRI_GPU_PROVER_ARCHIVER_ARCHIVE_AFTER_SECS="172800"
"#;
lock.set_env(config);

Expand Down
63 changes: 0 additions & 63 deletions core/lib/protobuf_config/src/house_keeper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,43 +12,6 @@ impl ProtoRepr for proto::HouseKeeper {
&self.l1_batch_metrics_reporting_interval_ms,
)
.context("l1_batch_metrics_reporting_interval_ms")?,
gpu_prover_queue_reporting_interval_ms: *required(
&self.gpu_prover_queue_reporting_interval_ms,
)
.context("gpu_prover_queue_reporting_interval_ms")?,
prover_job_retrying_interval_ms: *required(&self.prover_job_retrying_interval_ms)
.context("prover_job_retrying_interval_ms")?,
prover_stats_reporting_interval_ms: *required(&self.prover_stats_reporting_interval_ms)
.context("prover_stats_reporting_interval_ms")?,
witness_job_moving_interval_ms: *required(&self.witness_job_moving_interval_ms)
.context("witness_job_moving_interval_ms")?,
witness_generator_stats_reporting_interval_ms: *required(
&self.witness_generator_stats_reporting_interval_ms,
)
.context("witness_generator_stats_reporting_interval_ms")?,
prover_db_pool_size: *required(&self.prover_db_pool_size)
.context("prover_db_pool_size")?,
proof_compressor_job_retrying_interval_ms: *required(
&self.proof_compressor_job_retrying_interval_ms,
)
.context("proof_compressor_job_retrying_interval_ms")?,
witness_generator_job_retrying_interval_ms: *required(
&self.witness_generator_job_retrying_interval_ms,
)
.context("witness_generator_job_retrying_interval_ms")?,
proof_compressor_stats_reporting_interval_ms: *required(
&self.proof_compressor_stats_reporting_interval_ms,
)
.context("proof_compressor_stats_reporting_interval_ms")?,

// TODO(PLA-862): Make these 2 variables required
prover_job_archiver_archiving_interval_ms: self
.prover_job_archiver_archiving_interval_ms,
prover_job_archiver_archive_after_secs: self.prover_job_archiver_archive_after_secs,
fri_gpu_prover_archiver_archiving_interval_ms: self
.fri_gpu_prover_archiver_archiving_interval_ms,
fri_gpu_prover_archiver_archive_after_secs: self
.fri_gpu_prover_archiver_archive_after_secs,
})
}

Expand All @@ -57,32 +20,6 @@ impl ProtoRepr for proto::HouseKeeper {
l1_batch_metrics_reporting_interval_ms: Some(
this.l1_batch_metrics_reporting_interval_ms,
),
gpu_prover_queue_reporting_interval_ms: Some(
this.gpu_prover_queue_reporting_interval_ms,
),
prover_job_retrying_interval_ms: Some(this.prover_job_retrying_interval_ms),
prover_stats_reporting_interval_ms: Some(this.prover_stats_reporting_interval_ms),
witness_job_moving_interval_ms: Some(this.witness_job_moving_interval_ms),
witness_generator_stats_reporting_interval_ms: Some(
this.witness_generator_stats_reporting_interval_ms,
),
witness_generator_job_retrying_interval_ms: Some(
this.witness_generator_job_retrying_interval_ms,
),
prover_db_pool_size: Some(this.prover_db_pool_size),
proof_compressor_job_retrying_interval_ms: Some(
this.proof_compressor_job_retrying_interval_ms,
),
proof_compressor_stats_reporting_interval_ms: Some(
this.proof_compressor_stats_reporting_interval_ms,
),
prover_job_archiver_archiving_interval_ms: this
.prover_job_archiver_archiving_interval_ms,
prover_job_archiver_archive_after_secs: this.prover_job_archiver_archive_after_secs,
fri_gpu_prover_archiver_archiving_interval_ms: this
.fri_gpu_prover_archiver_archiving_interval_ms,
fri_gpu_prover_archiver_archive_after_secs: this
.fri_gpu_prover_archiver_archive_after_secs,
}
}
}
26 changes: 13 additions & 13 deletions core/lib/protobuf_config/src/proto/config/house_keeper.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@ package zksync.config.house_keeper;

message HouseKeeper {
optional uint64 l1_batch_metrics_reporting_interval_ms = 1; // required; ms
optional uint64 gpu_prover_queue_reporting_interval_ms = 2; // required; ms
optional uint64 prover_job_retrying_interval_ms = 3; // required; ms
optional uint64 prover_stats_reporting_interval_ms = 4; // required ms
optional uint64 witness_job_moving_interval_ms = 5; // required; ms
optional uint64 witness_generator_stats_reporting_interval_ms = 6; // required; ms
optional uint64 witness_generator_job_retrying_interval_ms = 9; // required; ms
optional uint32 prover_db_pool_size = 10; // required
optional uint64 proof_compressor_job_retrying_interval_ms = 12; // required; ms
optional uint64 proof_compressor_stats_reporting_interval_ms = 13; // required; ms
optional uint64 prover_job_archiver_archiving_interval_ms = 14; // optional; ms
optional uint64 prover_job_archiver_archive_after_secs = 15; // optional; seconds
optional uint64 fri_gpu_prover_archiver_archiving_interval_ms = 16; // optional; ms
optional uint64 fri_gpu_prover_archiver_archive_after_secs = 17; // optional; seconds
reserved 2; reserved "gpu_prover_queue_reporting_interval_ms";
reserved 3; reserved "prover_job_retrying_interval_ms";
reserved 4; reserved "prover_stats_reporting_interval_ms";
reserved 5; reserved "witness_job_moving_interval_ms";
reserved 6; reserved "witness_generator_stats_reporting_interval_ms";
reserved 9; reserved "witness_generator_job_retrying_interval_ms";
reserved 10; reserved "prover_db_pool_size";
reserved 12; reserved "proof_compressor_job_retrying_interval_ms";
reserved 13; reserved "proof_compressor_stats_reporting_interval_ms";
reserved 14; reserved "prover_job_archiver_archiving_interval_ms";
reserved 15; reserved "prover_job_archiver_archive_after_secs";
reserved 16; reserved "fri_gpu_prover_archiver_archiving_interval_ms";
reserved 17; reserved "fri_gpu_prover_archiver_archive_after_secs";
}
1 change: 0 additions & 1 deletion core/node/house_keeper/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ categories.workspace = true
vise.workspace = true
zksync_dal.workspace = true
zksync_shared_metrics.workspace = true
zksync_prover_dal.workspace = true
zksync_types.workspace = true
zksync_config.workspace = true

Expand Down
Loading
Loading