From 5d23b83527c6847224daaadb239f2b41d3ec4e9c Mon Sep 17 00:00:00 2001 From: welbon Date: Sun, 18 Feb 2024 18:16:39 +0800 Subject: [PATCH 01/16] Change new repository version for move language which has fixed argument check error that caused attack --- Cargo.lock | 78 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 52 ++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c7c99a783..a2e0c1897d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -978,7 +978,7 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "bytecode-interpreter-crypto" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "curve25519-dalek-fiat", @@ -5016,7 +5016,7 @@ checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" [[package]] name = "move-abigen" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5033,7 +5033,7 @@ dependencies = [ [[package]] name = "move-binary-format" version = "0.0.3" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "arbitrary", @@ -5049,12 +5049,12 @@ dependencies = [ [[package]] name = "move-borrow-graph" version = "0.0.1" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" [[package]] name = "move-bytecode-source-map" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5069,7 +5069,7 @@ dependencies = [ [[package]] name = "move-bytecode-utils" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "move-binary-format", @@ -5081,7 +5081,7 @@ dependencies = [ [[package]] name = "move-bytecode-verifier" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "fail", @@ -5094,7 +5094,7 @@ dependencies = [ [[package]] name = "move-bytecode-viewer" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "clap 3.2.23", @@ -5111,7 +5111,7 @@ dependencies = [ [[package]] name = "move-cli" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5157,7 +5157,7 @@ dependencies = [ [[package]] name = "move-command-line-common" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "difference", @@ -5174,7 +5174,7 @@ dependencies = [ [[package]] name = "move-compiler" version = "0.0.1" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5203,7 +5203,7 @@ dependencies = [ [[package]] name = "move-core-types" version = "0.0.4" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "arbitrary", @@ -5227,7 +5227,7 @@ dependencies = [ [[package]] name = "move-coverage" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5264,7 +5264,7 @@ dependencies = [ [[package]] name = "move-disassembler" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "clap 3.2.23", @@ -5282,7 +5282,7 @@ dependencies = [ [[package]] name = "move-docgen" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "codespan 0.11.1", @@ -5300,7 +5300,7 @@ dependencies = [ [[package]] name = "move-errmapgen" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5314,7 +5314,7 @@ dependencies = [ [[package]] name = "move-ir-compiler" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5333,7 +5333,7 @@ dependencies = [ [[package]] name = "move-ir-to-bytecode" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "codespan-reporting", @@ -5352,7 +5352,7 @@ dependencies = [ [[package]] name = "move-ir-to-bytecode-syntax" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "hex", @@ -5365,7 +5365,7 @@ dependencies = [ [[package]] name = "move-ir-types" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "hex", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "move-model" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "codespan 0.11.1", @@ -5405,7 +5405,7 @@ dependencies = [ [[package]] name = "move-package" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5489,7 +5489,7 @@ dependencies = [ [[package]] name = "move-prover" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "async-trait", @@ -5526,7 +5526,7 @@ dependencies = [ [[package]] name = "move-prover-boogie-backend" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "async-trait", @@ -5554,7 +5554,7 @@ dependencies = [ [[package]] name = "move-prover-test-utils" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "move-command-line-common", @@ -5565,7 +5565,7 @@ dependencies = [ [[package]] name = "move-read-write-set-types" version = "0.0.3" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "move-binary-format", @@ -5576,7 +5576,7 @@ dependencies = [ [[package]] name = "move-resource-viewer" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5591,7 +5591,7 @@ dependencies = [ [[package]] name = "move-stackless-bytecode" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "codespan 0.11.1", "codespan-reporting", @@ -5618,7 +5618,7 @@ dependencies = [ [[package]] name = "move-stackless-bytecode-interpreter" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bytecode-interpreter-crypto", @@ -5636,7 +5636,7 @@ dependencies = [ [[package]] name = "move-stdlib" version = "0.1.1" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "hex", @@ -5659,7 +5659,7 @@ dependencies = [ [[package]] name = "move-symbol-pool" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "once_cell", "serde 1.0.152", @@ -5668,7 +5668,7 @@ dependencies = [ [[package]] name = "move-table-extension" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "bcs", @@ -5685,7 +5685,7 @@ dependencies = [ [[package]] name = "move-transactional-test-runner" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "clap 3.2.23", @@ -5720,7 +5720,7 @@ dependencies = [ [[package]] name = "move-unit-test" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "better_any", @@ -5751,7 +5751,7 @@ dependencies = [ [[package]] name = "move-vm-runtime" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "better_any", "fail", @@ -5768,7 +5768,7 @@ dependencies = [ [[package]] name = "move-vm-test-utils" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "move-binary-format", @@ -5782,7 +5782,7 @@ dependencies = [ [[package]] name = "move-vm-types" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "bcs", "move-binary-format", @@ -7812,7 +7812,7 @@ dependencies = [ [[package]] name = "read-write-set" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "move-binary-format", @@ -7827,7 +7827,7 @@ dependencies = [ [[package]] name = "read-write-set-dynamic" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=127584f4cb1e2b4cd650f1e6c317250f5fccca49#127584f4cb1e2b4cd650f1e6c317250f5fccca49" +source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" dependencies = [ "anyhow", "move-binary-format", diff --git a/Cargo.toml b/Cargo.toml index 7d299ec4b8..cb919071f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -333,31 +333,31 @@ log4rs = { version = "1.2.0", features = ["background_rotation", "gzip"] } lru = "0.7.8" merkletree = { version = "0.22.1" } mirai-annotations = "1.10.1" -move-binary-format = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-bytecode-source-map = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-bytecode-verifier = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-cli = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-command-line-common = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-compiler = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-core-types = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-coverage = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-disassembler = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-docgen = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-errmapgen = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-ir-compiler = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-ir-types = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-model = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-package = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-prover = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-prover-test-utils = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-resource-viewer = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-stdlib = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-transactional-test-runner = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-unit-test = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49", features = ["table-extension"] } -move-vm-runtime = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-vm-types = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-table-extension = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49" } -move-vm-test-utils = { git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49", features = ["table-extension"] } +move-binary-format = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-bytecode-source-map = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-bytecode-verifier = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-cli = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-command-line-common = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-compiler = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-core-types = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-coverage = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-disassembler = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-docgen = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-errmapgen = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-ir-compiler = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-ir-types = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-model = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-package = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-prover = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-prover-test-utils = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-resource-viewer = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-stdlib = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-transactional-test-runner = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-unit-test = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16", features = ["table-extension"] } +move-vm-runtime = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-vm-types = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-table-extension = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } +move-vm-test-utils = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16", features = ["table-extension"] } names = { version = "0.14.0", default-features = false } network-api = { path = "network/api", package = "network-api" } @@ -522,7 +522,7 @@ unsigned-varint = { version = "0.6.0", features = [ "futures", "asynchronous_codec", ] } -vm = { package = "move-binary-format", git = "https://github.com/starcoinorg/move", rev = "127584f4cb1e2b4cd650f1e6c317250f5fccca49", features = [ +vm = { package = "move-binary-format", git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16", features = [ "fuzzing", ] } vm-status-translator = { path = "vm/vm-status-translator" } From e26d1c6b922bef137c1c6fc398d14aa31d267c00 Mon Sep 17 00:00:00 2001 From: welbon Date: Mon, 19 Feb 2024 23:45:06 +0800 Subject: [PATCH 02/16] add the db-export decode payload command code --- cmd/db-exporter/Cargo.toml | 2 + cmd/db-exporter/src/cmd_batch_execution.rs | 119 ---------- cmd/db-exporter/src/command_decode_payload.rs | 215 ++++++++++++++++++ cmd/db-exporter/src/command_progress.rs | 198 ++++++++++++++++ cmd/db-exporter/src/lib.rs | 35 ++- cmd/db-exporter/src/verify_header.rs | 28 ++- cmd/db-exporter/src/verify_module.rs | 23 +- 7 files changed, 490 insertions(+), 130 deletions(-) delete mode 100644 cmd/db-exporter/src/cmd_batch_execution.rs create mode 100644 cmd/db-exporter/src/command_decode_payload.rs create mode 100644 cmd/db-exporter/src/command_progress.rs diff --git a/cmd/db-exporter/Cargo.toml b/cmd/db-exporter/Cargo.toml index ee4b2dfb82..3b676ae918 100644 --- a/cmd/db-exporter/Cargo.toml +++ b/cmd/db-exporter/Cargo.toml @@ -32,9 +32,11 @@ starcoin-transaction-builder = { workspace = true } starcoin-types = { workspace = true } starcoin-vm-types = { workspace = true } starcoin-vm-runtime = { workspace = true } +starcoin-abi-decoder = { workspace = true } futures = { workspace = true } rayon = { workspace = true } num_cpus = { workspace = true } +chrono = { workspace = true } [package] authors = { workspace = true } diff --git a/cmd/db-exporter/src/cmd_batch_execution.rs b/cmd/db-exporter/src/cmd_batch_execution.rs deleted file mode 100644 index 7bb4c27a67..0000000000 --- a/cmd/db-exporter/src/cmd_batch_execution.rs +++ /dev/null @@ -1,119 +0,0 @@ -use anyhow::bail; -use indicatif::{ProgressBar, ProgressStyle}; -use rayon::prelude::*; -use std::{ - fs::File, - io::{BufRead, BufReader}, - path::PathBuf, - time::SystemTime, -}; - -pub struct CmdBatchExecution { - name: String, - file_path: PathBuf, - batch_size: usize, -} - -struct ExecutionResult { - succeed: usize, - failed: usize, -} - -impl ExecutionResult { - pub fn new(succeed: usize, failed: usize) -> ExecutionResult { - ExecutionResult { succeed, failed } - } -} - -impl CmdBatchExecution { - pub fn new(name: String, file_path: PathBuf, batch_size: usize) -> CmdBatchExecution { - Self { - file_path, - name, - batch_size, - } - } - - pub fn progress(self) -> anyhow::Result<()> - where - BodyT: BatchCmdExec - + Send - + Sync - + Clone - + serde::Serialize - + for<'a> serde::Deserialize<'a> - + 'static, - { - println!("Start progress task, batch_size: {:?}", self.batch_size); - - let mut start_time = SystemTime::now(); - let file_name = self.file_path.display().to_string(); - let reader = BufReader::new(File::open(file_name)?); - println!( - "Reading file process expire mini seconds time: {:?}", - SystemTime::now().duration_since(start_time)?.as_micros() - ); - - start_time = SystemTime::now(); - let lines = reader.lines().collect::, _>>()?; - - let all_items = lines - .par_iter() - .map(|line| Ok(serde_json::from_str::(line.as_str()))?) - .collect::, _>>()?; - - let progress_bar = ProgressBar::new(all_items.len() as u64).with_style( - ProgressStyle::default_bar() - .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), - ); - - println!( - "Reading lines from file expire time: {:?}", - SystemTime::now().duration_since(start_time)?.as_secs() - ); - - // It is necessary to divide all rows into subsets - // when reading them, - // so that they can be divided into several threads for the following operations - start_time = SystemTime::now(); - - let excution_result = all_items - .into_par_iter() - .chunks(self.batch_size) - .map(|item_vec| { - item_vec - .into_iter() - .map(|item| { - let (succeed, failed) = item.execute(); - progress_bar.inc(1); - ExecutionResult::new(succeed, failed.len()) - }) - .collect::>() - }) - .collect::>>(); - - let result = excution_result.into_iter().flatten().fold( - ExecutionResult { - succeed: 0, - failed: 0, - }, - |acc, result| ExecutionResult { - succeed: acc.succeed + result.succeed, - failed: acc.failed + result.failed, - }, - ); - - progress_bar.finish(); - - println!("verify {:?}, use time: {:?}, success modules: {}, error modules: {}, total modules: {}", - self.name, SystemTime::now().duration_since(start_time)?.as_secs(), result.succeed, result.failed, result.succeed + result.failed); - if result.failed > 0 { - bail!("verify block modules error"); - } - Ok(()) - } -} - -pub trait BatchCmdExec { - fn execute(&self) -> (usize, Vec); -} diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs new file mode 100644 index 0000000000..357e565cf1 --- /dev/null +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -0,0 +1,215 @@ +// Copyright (c) The Starcoin Core Contributors +// SPDX-License-Identifier: Apache-2.0 + +use crate::command_progress::{ + ParallelCommand, ParallelCommandFilter, ParallelCommandObserver, ParallelCommandProgress, +}; +use anyhow::Result; +use chrono::{TimeZone, Utc}; +use clap::Parser; +use csv::{Writer, WriterBuilder}; +use move_binary_format::errors::{Location, PartialVMError}; +use serde::Serialize; +use starcoin_crypto::hash::CryptoHash; +use starcoin_crypto::HashValue; +use starcoin_types::{block::Block, transaction::TransactionPayload}; +use starcoin_vm_types::errors::VMError; +use std::fs::File; +use std::sync::{Arc, Mutex}; +use std::{fmt::Debug, path::PathBuf}; +use starcoin_abi_decoder; +use starcoin_abi_decoder::{DecodedTransactionPayload}; +use starcoin_statedb::ChainStateDB; +use starcoin_storage::{Storage}; +use crate::init_db_obj; + +const DECODE_PAYLOAD_COMMAND_NAME: &str = "decode_payload_command"; + +#[derive(Debug, Parser)] +#[clap( +name = "decode-payload", +about = "Decode payload for given parameter and function name" +)] +pub struct DecodePayloadCommandOptions { + #[clap(long, short = 'i', parse(from_os_str))] + /// input file, like accounts.csv + pub input_path: PathBuf, + + #[clap(long, short = 'o', parse(from_os_str))] + /// output file, like accounts.csv + pub output_path: PathBuf, + + #[clap(long, short = 's')] + /// Signer filter + pub signer: Option, + + #[clap(long, short = 'f')] + /// function name for filter + pub func_name: Option, + + #[clap(long = "arg", multiple_values = true, number_of_values = 1)] + /// List of arguments for filter + pub args: Option>, + + #[clap(long, short = 't', multiple_values = true, number_of_values = 1)] + /// List of template arguments for filter + pub ty_args: Option>, +} + +#[derive(Debug)] +pub struct DecodePayloadCommandError { + pub block_number: u64, + pub txn_hash: HashValue, + pub error: VMError, +} + +impl DecodePayloadCommandError { + fn new_from_vm_error(error: VMError, block_number: u64, txn_hash: &HashValue) -> Self { + DecodePayloadCommandError { + block_number, + txn_hash: txn_hash.clone(), + error, + } + } + + fn new_from_partial_vm_error( + error: PartialVMError, + block_number: u64, + txn_hash: &HashValue, + ) -> Self { + DecodePayloadCommandError { + block_number, + txn_hash: txn_hash.clone(), + error: error.finish(Location::Undefined), + } + } +} + +#[derive(Serialize)] +pub struct CSVHeaders { + txn_hash: String, + signer: String, + func_name: String, + ty_args: String, + args: String, + timestamp: String, +} + +pub struct CommandDecodePayload { + out_path: PathBuf, + writer_mutex: Mutex>, + storage: Arc, +} + +impl ParallelCommandObserver for CommandDecodePayload { + fn before_progress(&self) -> Result<()> { + Ok(()) + } + + fn after_progress(&self) -> Result<()> { + let mut writer = self.writer_mutex.lock().unwrap(); + writer.flush()?; + Ok(()) + } +} + +impl ParallelCommand for Block { + fn execute(&self, command: &CommandDecodePayload) -> (usize, Vec) { + let mut errors = vec![]; + let mut success_module_size = 0; + + let datetime = Utc.timestamp_opt(self.header.timestamp() as i64, 0); + let formatted_date = datetime.unwrap().format("%Y-%m-%d %H:%M:%s").to_string(); + + let root = self.header.state_root(); + let statedb = ChainStateDB::new(command.storage, Some(root)); + + for txn in self.transactions() { + let signer = txn.sender().to_string(); + let decoded_txn_payload = starcoin_abi_decoder::decode_txn_payload( + &statedb, txn.payload(), + ).expect("Decode transaction payload failed!"); + + match decoded_txn_payload { + DecodedTransactionPayload::ScriptFunction(payload) => { + let mut writer = command.writer_mutex.lock().unwrap(); + writer.serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + signer, + func_name: format!("{}::{}", payload.module, payload.function), + //ty_args: payload.ty_args.iter().map(|a| a.to_string() + ",").collect(), + //args: payload.args.iter().map(|dv| format!("{},", dv.0.as_str()).collect(), + ty_args: payload.ty_args.iter().map(|a| a.to_string()).collect::>().join(","), + args: payload.args.iter().map(|a| a.0.to_string()).collect::>().join(","), + timestamp: formatted_date.clone(), + }).expect("Write into CSV failed!") + } + DecodedTransactionPayload::Script(_) | DecodedTransactionPayload::Package(_) => (), + } + } + (success_module_size, errors) + } + + fn before_command(&self, _cmd: &CommandDecodePayload) -> Result<()> { + Ok(()) + } + + fn after_command(&self, _cmd: &CommandDecodePayload) -> Result<()> { + Ok(()) + } + + /// + /// Check whether the conditions are met from the list of all transactions in a block, + /// and return false if any condition is met. + /// + fn matched(&self, filters: Option) -> bool { + filters.as_ref().map_or_else(|| true, |f| { + self.transactions().iter().any(|txn| match txn.payload() { + TransactionPayload::ScriptFunction(payload) => { + f.match_signer(&txn.sender().to_string()) + || f.match_func_name(payload.function().as_str()) + || f.match_ty_args(&payload.ty_args().to_vec()) + || f.match_args(&payload.args().to_vec()) + } + _ => true, + }) + }) + } +} + + +pub fn decode_payload( + input_path: PathBuf, + out_path: PathBuf, + db_path: PathBuf, + filter: Option, +) -> Result<()> { + let file = WriterBuilder::new().from_path(out_path.as_ref())?; + let writer_mutex = Mutex::new(file); + + let command = Arc::new(CommandDecodePayload { + out_path, + writer_mutex, + storage: init_db_obj(db_path)?, + }); + + ParallelCommandProgress::new( + String::from(DECODE_PAYLOAD_COMMAND_NAME), + input_path, + num_cpus::get(), + filter, + Some(command.clone() as Arc), + ).progress::(&command) +} + +#[test] +pub fn test_decode_payload() { + let mut workspace = PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet"); + let mut input = workspace.clone(); + input.push("grep-'LocalPool'.json"); + + let mut output = workspace.clone(); + output.push("output.csv"); + + decode_payload(input, output, PathBuf::from(""), None)?; +} diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs new file mode 100644 index 0000000000..56530e9788 --- /dev/null +++ b/cmd/db-exporter/src/command_progress.rs @@ -0,0 +1,198 @@ +use anyhow::{bail, Result}; +use indicatif::{ProgressBar, ProgressStyle}; +use rayon::prelude::*; +use starcoin_vm_types::language_storage::TypeTag; +use std::sync::Arc; +use std::{ + fs::File, + io::{BufRead, BufReader}, + path::PathBuf, + time::SystemTime, +}; + +struct CommandResult { + succeed: usize, + failed: usize, +} + +impl CommandResult { + pub fn new(succeed: usize, failed: usize) -> CommandResult { + CommandResult { succeed, failed } + } +} + +pub struct ParallelCommandFilter { + signer: Option, + func_name: Option, + // function name for filter, none for all + ty_args: Option>, + // template parameter for filter, none for all + args: Option>, // arguments type for filter, none for all +} + +impl ParallelCommandFilter { + fn new( + signer: Option, + func_name: Option, + ty_args: Option>, + args: Option>, + ) -> Option { + if func_name.is_some() || ty_args.is_some() || args.is_some() { + Some(ParallelCommandFilter { + signer, + func_name, + ty_args, + args, + }) + } else { + None + } + } + + pub fn match_signer(&self, signer: &str) -> bool { + self.signer.as_ref().map_or(false, |n| n == signer) + } + + pub fn match_func_name(&self, func_name: &str) -> bool { + self.func_name.as_ref().map_or(false, |n| n == func_name) + } + + pub fn match_ty_args(&self, _ty_args: &Vec) -> bool { + // TODO(Bob): To Compare + true + } + + pub fn match_args(&self, _args: &Vec>) -> bool { + // TODO(Bob): To Compare + true + } +} + +pub trait ParallelCommandObserver { + fn before_progress(&self) -> Result<()>; + fn after_progress(&self) -> Result<()>; +} + +pub struct ParallelCommandProgress { + name: String, + file_path: PathBuf, + parallel_level: usize, + filter: Option, + obs: Option>, +} + +impl ParallelCommandProgress { + pub fn new( + name: String, + file_path: PathBuf, + parallel_level: usize, + filter: Option, + obs: Option>, + ) -> ParallelCommandProgress { + Self { + file_path, + name, + parallel_level, + filter, + obs, + } + } + + pub fn progress(self, command: &CommandT) -> Result<()> + where + BodyT: ParallelCommand + + Send + + Sync + + Clone + + serde::Serialize + + for<'a> serde::Deserialize<'a> + + 'static, + { + println!("Start progress task, batch_size: {:?}", self.parallel_level); + + let mut start_time = SystemTime::now(); + let file_name = self.file_path.display().to_string(); + let reader = BufReader::new(File::open(file_name)?); + println!( + "Reading file process expire mini seconds time: {:?}", + SystemTime::now().duration_since(start_time)?.as_micros() + ); + + start_time = SystemTime::now(); + let lines = reader.lines().collect::, _>>()?; + + if let Some(observer) = &self.obs { + observer.before_progress()?; + } + + let all_items = lines + .par_iter() + .map(|line| Ok(serde_json::from_str::(line.as_str()))?) + .filter(|item| item.unwrap().matched(self.filters)) + .collect::, _>>()?; + + let progress_bar = ProgressBar::new(all_items.len() as u64).with_style( + ProgressStyle::default_bar() + .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), + ); + + println!( + "Reading lines from file expire time: {:?}", + SystemTime::now().duration_since(start_time)?.as_secs() + ); + + // It is necessary to divide all rows into subsets + // when reading them, + // so that they can be divided into several threads for the following operations + start_time = SystemTime::now(); + + let excution_result = all_items + .into_par_iter() + .chunks(self.parallel_level) + .map(|item_vec| { + item_vec + .into_iter() + .map(|item| { + let (succeed, failed) = item.execute(command); + progress_bar.inc(1); + CommandResult::new(succeed, failed.len()) + }) + .collect::>() + }) + .collect::>>(); + + let result = excution_result.into_iter().flatten().fold( + CommandResult { + succeed: 0, + failed: 0, + }, + |acc, result| CommandResult { + succeed: acc.succeed + result.succeed, + failed: acc.failed + result.failed, + }, + ); + + progress_bar.finish(); + + println!("verify {:?}, use time: {:?}, success modules: {}, error modules: {}, total modules: {}", + self.name, SystemTime::now().duration_since(start_time)?.as_secs(), result.succeed, result.failed, result.succeed + result.failed); + if result.failed > 0 { + bail!("verify block modules error"); + } + if let Some(observer) = &self.obs { + observer.after_progress()?; + } + + Ok(()) + } +} + +pub trait ParallelCommand { + fn execute(&self, cmd: &CommandT) -> (usize, Vec); + + fn before_command(&self, cmd: &CommandT) -> Result<()>; + + fn after_command(&self, cmd: &CommandT) -> Result<()>; + + fn matched(&self, filter: Option) -> bool; +} diff --git a/cmd/db-exporter/src/lib.rs b/cmd/db-exporter/src/lib.rs index 99e57915f1..a4fab8b02f 100644 --- a/cmd/db-exporter/src/lib.rs +++ b/cmd/db-exporter/src/lib.rs @@ -1,6 +1,39 @@ // Copyright (c) The Starcoin Core Contributors // SPDX-License-Identifier: Apache-2.0 -pub mod cmd_batch_execution; +use std::path::PathBuf; +use std::sync::Arc; +use starcoin_storage::{ + db_storage::{ + DBStorage + }, + Storage, + StorageVersion, + storage::StorageInstance, + cache_storage::CacheStorage +}; +use anyhow::Result; + +pub mod command_decode_payload; +pub mod command_progress; pub mod verify_header; pub mod verify_module; + + +pub fn init_db_obj( + db_path: PathBuf, +) -> Result> { + let db_storage = DBStorage::open_with_cfs( + db_path.join("starcoindb/db/starcoindb"), + StorageVersion::current_version() + .get_column_family_names() + .to_vec(), + true, + Default::default(), + None, + )?; + Ok(Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance( + CacheStorage::new(None), + db_storage, + ))?)) +} \ No newline at end of file diff --git a/cmd/db-exporter/src/verify_header.rs b/cmd/db-exporter/src/verify_header.rs index b49baffcdc..8c42003e9f 100644 --- a/cmd/db-exporter/src/verify_header.rs +++ b/cmd/db-exporter/src/verify_header.rs @@ -5,7 +5,7 @@ use clap::Parser; use starcoin_consensus::{Consensus, G_CRYPTONIGHT}; use std::path::PathBuf; -use crate::cmd_batch_execution::{BatchCmdExec, CmdBatchExecution}; +use crate::command_progress::{ParallelCommand, ParallelCommandFilter, ParallelCommandProgress}; use starcoin_types::block::Block; #[derive(Debug, Parser)] @@ -28,12 +28,18 @@ pub struct VerifyHeaderError { pub struct VerifyHeaderCmdType; pub fn verify_header_via_export_file(path: PathBuf, batch_size: usize) -> anyhow::Result<()> { - let batch_cmd = CmdBatchExecution::new(String::from("verify_block_header"), path, batch_size); - batch_cmd.progress::() + let batch_cmd = ParallelCommandProgress::new( + String::from("verify_block_header"), + path, + batch_size, + None, + None, + ); + batch_cmd.progress::(&VerifyHeaderCmdType {}) } -impl BatchCmdExec for Block { - fn execute(&self) -> (usize, Vec) { +impl ParallelCommand for Block { + fn execute(&self, _cmd: &VerifyHeaderCmdType) -> (usize, Vec) { let ret = G_CRYPTONIGHT.verify_header_difficulty(self.header.difficulty(), &self.header); match ret { Ok(_) => (1, vec![]), @@ -48,4 +54,16 @@ impl BatchCmdExec for Block { } } } + + fn before_command(&self, _cmd: &VerifyHeaderCmdType) -> anyhow::Result<()> { + Ok(()) + } + + fn after_command(&self, _cmd: &VerifyHeaderCmdType) -> anyhow::Result<()> { + Ok(()) + } + + fn matched(&self, _filter: Option) -> bool { + true + } } diff --git a/cmd/db-exporter/src/verify_module.rs b/cmd/db-exporter/src/verify_module.rs index 687cb74bbb..4c36ac4d96 100644 --- a/cmd/db-exporter/src/verify_module.rs +++ b/cmd/db-exporter/src/verify_module.rs @@ -5,7 +5,7 @@ use starcoin_types::{block::Block, transaction::TransactionPayload}; use starcoin_vm_types::{errors::VMError, file_format::CompiledModule}; use std::{fmt::Debug, path::PathBuf}; //use starcoin_accumulator::node::AccumulatorStoreType::Block; -use crate::cmd_batch_execution::{BatchCmdExec, CmdBatchExecution}; +use crate::command_progress::{ParallelCommand, ParallelCommandFilter, ParallelCommandProgress}; #[derive(Debug, Parser)] #[clap( @@ -27,8 +27,8 @@ pub struct VerifyModuleError { pub struct VerifyModulesType; -impl BatchCmdExec for Block { - fn execute(&self) -> (usize, Vec) { +impl ParallelCommand for Block { + fn execute(&self, _cmd: &VerifyModulesType) -> (usize, Vec) { let mut errors = vec![]; let mut success_modules = 0; let block = self; @@ -72,6 +72,18 @@ impl BatchCmdExec for Block { } (success_modules, errors) } + + fn before_command(&self, _cmd: &VerifyModulesType) -> anyhow::Result<()> { + Ok(()) + } + + fn after_command(&self, _cmd: &VerifyModulesType) -> anyhow::Result<()> { + Ok(()) + } + + fn matched(&self, _filter: Option) -> bool { + true + } } // fn verify_block_modules(block: Block) -> (usize, Vec) { @@ -119,8 +131,9 @@ impl BatchCmdExec for Block { // } pub fn verify_modules_via_export_file(input_path: PathBuf) -> anyhow::Result<()> { - let batch_cmd = CmdBatchExecution::new(String::from("verify_module"), input_path, 10); - batch_cmd.progress::() + let batch_cmd = + ParallelCommandProgress::new(String::from("verify_module"), input_path, 10, None, None); + batch_cmd.progress::(&VerifyModulesType {}) // let start_time = SystemTime::now(); // let file_name = input_path.display().to_string(); // let reader = BufReader::new(File::open(input_path)?); From 0a2fd42fb0605336cc7fd6975979bbf1d46729f1 Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 10:19:45 +0800 Subject: [PATCH 03/16] fixed compile error --- cmd/db-exporter/src/command_decode_payload.rs | 105 +++++++++--------- cmd/db-exporter/src/command_progress.rs | 27 +++-- cmd/db-exporter/src/lib.rs | 27 ++--- cmd/db-exporter/src/verify_header.rs | 10 +- cmd/db-exporter/src/verify_module.rs | 10 +- 5 files changed, 80 insertions(+), 99 deletions(-) diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs index 357e565cf1..6fa8aad78d 100644 --- a/cmd/db-exporter/src/command_decode_payload.rs +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -4,31 +4,30 @@ use crate::command_progress::{ ParallelCommand, ParallelCommandFilter, ParallelCommandObserver, ParallelCommandProgress, }; +use crate::init_db_obj; use anyhow::Result; use chrono::{TimeZone, Utc}; use clap::Parser; use csv::{Writer, WriterBuilder}; use move_binary_format::errors::{Location, PartialVMError}; use serde::Serialize; -use starcoin_crypto::hash::CryptoHash; -use starcoin_crypto::HashValue; +use starcoin_abi_decoder; +use starcoin_abi_decoder::DecodedTransactionPayload; +use starcoin_crypto::{hash::CryptoHash, HashValue}; +use starcoin_statedb::ChainStateDB; +use starcoin_storage::Storage; use starcoin_types::{block::Block, transaction::TransactionPayload}; use starcoin_vm_types::errors::VMError; use std::fs::File; use std::sync::{Arc, Mutex}; use std::{fmt::Debug, path::PathBuf}; -use starcoin_abi_decoder; -use starcoin_abi_decoder::{DecodedTransactionPayload}; -use starcoin_statedb::ChainStateDB; -use starcoin_storage::{Storage}; -use crate::init_db_obj; const DECODE_PAYLOAD_COMMAND_NAME: &str = "decode_payload_command"; #[derive(Debug, Parser)] #[clap( -name = "decode-payload", -about = "Decode payload for given parameter and function name" + name = "decode-payload", + about = "Decode payload for given parameter and function name" )] pub struct DecodePayloadCommandOptions { #[clap(long, short = 'i', parse(from_os_str))] @@ -96,7 +95,6 @@ pub struct CSVHeaders { } pub struct CommandDecodePayload { - out_path: PathBuf, writer_mutex: Mutex>, storage: Arc, } @@ -115,80 +113,84 @@ impl ParallelCommandObserver for CommandDecodePayload { impl ParallelCommand for Block { fn execute(&self, command: &CommandDecodePayload) -> (usize, Vec) { - let mut errors = vec![]; - let mut success_module_size = 0; + // let errors = vec![]; + // let mut success_module_size = 0; let datetime = Utc.timestamp_opt(self.header.timestamp() as i64, 0); let formatted_date = datetime.unwrap().format("%Y-%m-%d %H:%M:%s").to_string(); let root = self.header.state_root(); - let statedb = ChainStateDB::new(command.storage, Some(root)); + let statedb = ChainStateDB::new(command.storage.clone(), Some(root)); for txn in self.transactions() { let signer = txn.sender().to_string(); - let decoded_txn_payload = starcoin_abi_decoder::decode_txn_payload( - &statedb, txn.payload(), - ).expect("Decode transaction payload failed!"); + let decoded_txn_payload = + starcoin_abi_decoder::decode_txn_payload(&statedb, txn.payload()) + .expect("Decode transaction payload failed!"); match decoded_txn_payload { DecodedTransactionPayload::ScriptFunction(payload) => { let mut writer = command.writer_mutex.lock().unwrap(); - writer.serialize(CSVHeaders { - txn_hash: txn.hash().to_string(), - signer, - func_name: format!("{}::{}", payload.module, payload.function), - //ty_args: payload.ty_args.iter().map(|a| a.to_string() + ",").collect(), - //args: payload.args.iter().map(|dv| format!("{},", dv.0.as_str()).collect(), - ty_args: payload.ty_args.iter().map(|a| a.to_string()).collect::>().join(","), - args: payload.args.iter().map(|a| a.0.to_string()).collect::>().join(","), - timestamp: formatted_date.clone(), - }).expect("Write into CSV failed!") + writer + .serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + signer, + func_name: format!("{}::{}", payload.module, payload.function), + ty_args: payload + .ty_args + .iter() + .map(|a| a.to_string()) + .collect::>() + .join(","), + args: payload + .args + .iter() + .map(|a| a.0.to_string()) + .collect::>() + .join(","), + timestamp: formatted_date.clone(), + }) + .expect("Write into CSV failed!") } DecodedTransactionPayload::Script(_) | DecodedTransactionPayload::Package(_) => (), } } - (success_module_size, errors) - } - - fn before_command(&self, _cmd: &CommandDecodePayload) -> Result<()> { - Ok(()) - } - - fn after_command(&self, _cmd: &CommandDecodePayload) -> Result<()> { - Ok(()) + //(success_module_size, errors) + (0, vec![]) } /// /// Check whether the conditions are met from the list of all transactions in a block, /// and return false if any condition is met. /// - fn matched(&self, filters: Option) -> bool { - filters.as_ref().map_or_else(|| true, |f| { - self.transactions().iter().any(|txn| match txn.payload() { - TransactionPayload::ScriptFunction(payload) => { - f.match_signer(&txn.sender().to_string()) - || f.match_func_name(payload.function().as_str()) - || f.match_ty_args(&payload.ty_args().to_vec()) - || f.match_args(&payload.args().to_vec()) - } - _ => true, - }) - }) + fn matched(&self, filters: &Option) -> bool { + filters.as_ref().map_or_else( + || true, + |f| { + self.transactions().iter().any(|txn| match txn.payload() { + TransactionPayload::ScriptFunction(payload) => { + f.match_signer(&txn.sender().to_string()) + || f.match_func_name(payload.function().as_str()) + || f.match_ty_args(&payload.ty_args().to_vec()) + || f.match_args(&payload.args().to_vec()) + } + _ => true, + }) + }, + ) } } - pub fn decode_payload( input_path: PathBuf, out_path: PathBuf, db_path: PathBuf, filter: Option, ) -> Result<()> { - let file = WriterBuilder::new().from_path(out_path.as_ref())?; + let file = WriterBuilder::new().from_path(out_path.clone())?; let writer_mutex = Mutex::new(file); let command = Arc::new(CommandDecodePayload { - out_path, writer_mutex, storage: init_db_obj(db_path)?, }); @@ -199,7 +201,8 @@ pub fn decode_payload( num_cpus::get(), filter, Some(command.clone() as Arc), - ).progress::(&command) + ) + .progress::(&command) } #[test] diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 56530e9788..035b3107c2 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -23,11 +23,9 @@ impl CommandResult { pub struct ParallelCommandFilter { signer: Option, - func_name: Option, - // function name for filter, none for all - ty_args: Option>, - // template parameter for filter, none for all - args: Option>, // arguments type for filter, none for all + func_name: Option, // function name for filter, none for all + ty_args: Option>, // template parameter for filter, none for all + args: Option>, // arguments type for filter, none for all } impl ParallelCommandFilter { @@ -98,7 +96,7 @@ impl ParallelCommandProgress { } } - pub fn progress(self, command: &CommandT) -> Result<()> + pub fn progress(self, command: &CommandT) -> Result<()> where BodyT: ParallelCommand + Send @@ -128,7 +126,16 @@ impl ParallelCommandProgress { let all_items = lines .par_iter() .map(|line| Ok(serde_json::from_str::(line.as_str()))?) - .filter(|item| item.unwrap().matched(self.filters)) + .filter(|item| match item { + Ok(i) => { + if i.matched(&self.filter) { + true + } else { + false + } + } + Err(_e) => false, + }) .collect::, _>>()?; let progress_bar = ProgressBar::new(all_items.len() as u64).with_style( @@ -190,9 +197,5 @@ impl ParallelCommandProgress { pub trait ParallelCommand { fn execute(&self, cmd: &CommandT) -> (usize, Vec); - fn before_command(&self, cmd: &CommandT) -> Result<()>; - - fn after_command(&self, cmd: &CommandT) -> Result<()>; - - fn matched(&self, filter: Option) -> bool; + fn matched(&self, filter: &Option) -> bool; } diff --git a/cmd/db-exporter/src/lib.rs b/cmd/db-exporter/src/lib.rs index a4fab8b02f..a4d5215d21 100644 --- a/cmd/db-exporter/src/lib.rs +++ b/cmd/db-exporter/src/lib.rs @@ -1,28 +1,20 @@ // Copyright (c) The Starcoin Core Contributors // SPDX-License-Identifier: Apache-2.0 -use std::path::PathBuf; -use std::sync::Arc; +use anyhow::Result; use starcoin_storage::{ - db_storage::{ - DBStorage - }, - Storage, + cache_storage::CacheStorage, db_storage::DBStorage, storage::StorageInstance, Storage, StorageVersion, - storage::StorageInstance, - cache_storage::CacheStorage }; -use anyhow::Result; +use std::path::PathBuf; +use std::sync::Arc; pub mod command_decode_payload; pub mod command_progress; pub mod verify_header; pub mod verify_module; - -pub fn init_db_obj( - db_path: PathBuf, -) -> Result> { +pub fn init_db_obj(db_path: PathBuf) -> Result> { let db_storage = DBStorage::open_with_cfs( db_path.join("starcoindb/db/starcoindb"), StorageVersion::current_version() @@ -32,8 +24,7 @@ pub fn init_db_obj( Default::default(), None, )?; - Ok(Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance( - CacheStorage::new(None), - db_storage, - ))?)) -} \ No newline at end of file + Ok(Arc::new(Storage::new( + StorageInstance::new_cache_and_db_instance(CacheStorage::new(None), db_storage), + )?)) +} diff --git a/cmd/db-exporter/src/verify_header.rs b/cmd/db-exporter/src/verify_header.rs index 8c42003e9f..983c73e7f6 100644 --- a/cmd/db-exporter/src/verify_header.rs +++ b/cmd/db-exporter/src/verify_header.rs @@ -55,15 +55,7 @@ impl ParallelCommand for Block { } } - fn before_command(&self, _cmd: &VerifyHeaderCmdType) -> anyhow::Result<()> { - Ok(()) - } - - fn after_command(&self, _cmd: &VerifyHeaderCmdType) -> anyhow::Result<()> { - Ok(()) - } - - fn matched(&self, _filter: Option) -> bool { + fn matched(&self, _filter: &Option) -> bool { true } } diff --git a/cmd/db-exporter/src/verify_module.rs b/cmd/db-exporter/src/verify_module.rs index 4c36ac4d96..ed76ff0b89 100644 --- a/cmd/db-exporter/src/verify_module.rs +++ b/cmd/db-exporter/src/verify_module.rs @@ -73,15 +73,7 @@ impl ParallelCommand for Block { (success_modules, errors) } - fn before_command(&self, _cmd: &VerifyModulesType) -> anyhow::Result<()> { - Ok(()) - } - - fn after_command(&self, _cmd: &VerifyModulesType) -> anyhow::Result<()> { - Ok(()) - } - - fn matched(&self, _filter: Option) -> bool { + fn matched(&self, _filter: &Option) -> bool { true } } From 985d02d68055271746c02613fa34ac0cf67ae31b Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 13:55:02 +0800 Subject: [PATCH 04/16] fixed build error --- cmd/db-exporter/src/command_decode_payload.rs | 46 ++-- cmd/db-exporter/src/command_progress.rs | 222 ++++++++++++++---- cmd/db-exporter/src/lib.rs | 23 -- cmd/db-exporter/src/verify_header.rs | 12 +- cmd/db-exporter/src/verify_module.rs | 19 +- 5 files changed, 230 insertions(+), 92 deletions(-) diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs index 6fa8aad78d..57cd8d438b 100644 --- a/cmd/db-exporter/src/command_decode_payload.rs +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -3,8 +3,8 @@ use crate::command_progress::{ ParallelCommand, ParallelCommandFilter, ParallelCommandObserver, ParallelCommandProgress, + ParallelCommandReadBlockFromDB, }; -use crate::init_db_obj; use anyhow::Result; use chrono::{TimeZone, Utc}; use clap::Parser; @@ -13,6 +13,8 @@ use move_binary_format::errors::{Location, PartialVMError}; use serde::Serialize; use starcoin_abi_decoder; use starcoin_abi_decoder::DecodedTransactionPayload; +use starcoin_config::BuiltinNetworkID::Barnard; +use starcoin_config::ChainNetwork; use starcoin_crypto::{hash::CryptoHash, HashValue}; use starcoin_statedb::ChainStateDB; use starcoin_storage::Storage; @@ -31,13 +33,19 @@ const DECODE_PAYLOAD_COMMAND_NAME: &str = "decode_payload_command"; )] pub struct DecodePayloadCommandOptions { #[clap(long, short = 'i', parse(from_os_str))] - /// input file, like accounts.csv + /// Db path pub input_path: PathBuf, #[clap(long, short = 'o', parse(from_os_str))] /// output file, like accounts.csv pub output_path: PathBuf, + #[clap(long)] + pub start_height: Option, + + #[clap(long)] + pub end_height: Option, + #[clap(long, short = 's')] /// Signer filter pub signer: Option, @@ -111,7 +119,7 @@ impl ParallelCommandObserver for CommandDecodePayload { } } -impl ParallelCommand for Block { +impl ParallelCommand for Block { fn execute(&self, command: &CommandDecodePayload) -> (usize, Vec) { // let errors = vec![]; // let mut success_module_size = 0; @@ -184,35 +192,41 @@ impl ParallelCommand for pub fn decode_payload( input_path: PathBuf, out_path: PathBuf, - db_path: PathBuf, + start_height: Option, + end_height: Option, filter: Option, ) -> Result<()> { let file = WriterBuilder::new().from_path(out_path.clone())?; let writer_mutex = Mutex::new(file); + let (dbreader, storage) = ParallelCommandReadBlockFromDB::new( + input_path, + ChainNetwork::from(Barnard), + start_height.unwrap_or(0), + end_height.unwrap_or(0), + )?; let command = Arc::new(CommandDecodePayload { writer_mutex, - storage: init_db_obj(db_path)?, + storage, }); ParallelCommandProgress::new( String::from(DECODE_PAYLOAD_COMMAND_NAME), - input_path, num_cpus::get(), + Arc::new(dbreader), filter, Some(command.clone() as Arc), ) - .progress::(&command) + .progress::(&command) } #[test] -pub fn test_decode_payload() { - let mut workspace = PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet"); - let mut input = workspace.clone(); - input.push("grep-'LocalPool'.json"); - - let mut output = workspace.clone(); - output.push("output.csv"); - - decode_payload(input, output, PathBuf::from(""), None)?; +pub fn test_decode_payload() -> Result<()> { + decode_payload( + PathBuf::from("~/.starcoin/barnard"), + PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet/output.csv"), + None, + None, + None, + ) } diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 035b3107c2..00035e65fe 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -1,14 +1,18 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, format_err, Result}; use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::*; +use starcoin_chain::{BlockChain, ChainReader}; +use starcoin_config::ChainNetwork; +use starcoin_genesis::Genesis; +use starcoin_storage::cache_storage::CacheStorage; +use starcoin_storage::db_storage::DBStorage; +use starcoin_storage::storage::StorageInstance; +use starcoin_storage::{Storage, StorageVersion}; +use starcoin_types::block::{Block, BlockNumber}; use starcoin_vm_types::language_storage::TypeTag; use std::sync::Arc; -use std::{ - fs::File, - io::{BufRead, BufReader}, - path::PathBuf, - time::SystemTime, -}; +use std::{fs::File, io::{BufRead, BufReader}, path::PathBuf, time::SystemTime}; +use std::io::{Seek, SeekFrom}; struct CommandResult { succeed: usize, @@ -23,9 +27,11 @@ impl CommandResult { pub struct ParallelCommandFilter { signer: Option, - func_name: Option, // function name for filter, none for all - ty_args: Option>, // template parameter for filter, none for all - args: Option>, // arguments type for filter, none for all + func_name: Option, + // function name for filter, none for all + ty_args: Option>, + // template parameter for filter, none for all + args: Option>, // arguments type for filter, none for all } impl ParallelCommandFilter { @@ -66,15 +72,140 @@ impl ParallelCommandFilter { } } -pub trait ParallelCommandObserver { - fn before_progress(&self) -> Result<()>; - fn after_progress(&self) -> Result<()>; +pub struct ParallelCommandReadBodyFromExportLine { + file: File, + line_count: u64, +} + +impl ParallelCommandReadBodyFromExportLine { + + fn count_lines(reader: &mut BufReader) -> Result { + let line_count = reader.lines().count(); + reader.seek(SeekFrom::Start(0))?; + Ok(line_count as u64) + } + + pub fn new(input_path: PathBuf) -> Result { + let file = File::open(input_path.display().to_string())?; + let line_count = ParallelCommandReadBodyFromExportLine::count_lines(&mut BufReader::new(file.try_clone()?))?; + Ok(Self { + file, + line_count, + }) + } +} + +impl ParallelCommandBlockReader for ParallelCommandReadBodyFromExportLine { + fn get_progress_interval(&self) -> u64 { + self.line_count + } + + fn read(&self) -> Result> { + let reader = BufReader::new(self.file.try_clone()?); + let lines = reader.lines().collect::, _>>()?; + Ok(lines + .par_iter() + .map(|line| Ok(serde_json::from_str::(line.as_str()))?) + .collect::, _>>()?) + } +} + +pub struct ParallelCommandReadBlockFromDB { + start_num: u64, + end_num: u64, + chain: Arc, +} + +const BLOCK_GAP: u64 = 1000; + +impl ParallelCommandReadBlockFromDB { + pub fn new( + input_path: PathBuf, + net: ChainNetwork, + start: u64, + end: u64, + ) -> Result<(Self, Arc)> { + let storage = Self::init_db_obj(input_path.clone()).expect("Failed to initialize db"); + let (chain_info, _) = + Genesis::init_and_check_storage(&net, storage.clone(), input_path.as_ref()) + .expect("Failed init_and_check_storage"); + let chain = BlockChain::new(net.time_service(), chain_info.head().id(), storage.clone(), None) + .expect("Failed to initialize block chain"); + + let cur_num = chain.status().head().number(); + + let (start_num, end_num) = if start != 0 && end == 0 { + (0, cur_num) + } else { + let end = if cur_num > end + BLOCK_GAP { + end + } else if cur_num > BLOCK_GAP { + cur_num - BLOCK_GAP + } else { + end + }; + (start, end) + }; + + if start > cur_num || start > end { + return Err(format_err!( + "cur_num {} start {} end {} illegal", + cur_num, + start, + end + )); + }; + + Ok(( + Self { + start_num, + end_num, + chain: Arc::new(chain), + }, + storage, + )) + } + fn init_db_obj(db_path: PathBuf) -> Result> { + let db_storage = DBStorage::open_with_cfs( + db_path.join("starcoindb/db/starcoindb"), + StorageVersion::current_version() + .get_column_family_names() + .to_vec(), + true, + Default::default(), + None, + )?; + Ok(Arc::new(Storage::new( + StorageInstance::new_cache_and_db_instance(CacheStorage::new(None), db_storage), + )?)) + } +} + +impl ParallelCommandBlockReader for ParallelCommandReadBlockFromDB { + fn get_progress_interval(&self) -> u64 { + self.end_num - self.start_num + } + + fn read(&self) -> Result> { + let ret = (self.start_num..=self.end_num) + .collect::>() + .into_iter() + .map(|num| { + // progress_bar.set_message(format!("load block {}", num)); + // progress_bar.inc(1); + self.chain.get_block_by_number(num).ok()? + }) + .filter(|block| block.is_some()) + .map(|block| block.unwrap()) + .collect(); + Ok(ret) + } } pub struct ParallelCommandProgress { name: String, - file_path: PathBuf, parallel_level: usize, + block_reader: Arc, filter: Option, obs: Option>, } @@ -82,67 +213,60 @@ pub struct ParallelCommandProgress { impl ParallelCommandProgress { pub fn new( name: String, - file_path: PathBuf, parallel_level: usize, + reader: Arc, filter: Option, obs: Option>, ) -> ParallelCommandProgress { Self { - file_path, name, + block_reader: reader.clone(), parallel_level, filter, obs, } } - pub fn progress(self, command: &CommandT) -> Result<()> + pub fn progress(self, command: &CommandT) -> Result<()> where - BodyT: ParallelCommand - + Send - + Sync - + Clone - + serde::Serialize - + for<'a> serde::Deserialize<'a> - + 'static, + Block: ParallelCommand, { println!("Start progress task, batch_size: {:?}", self.parallel_level); let mut start_time = SystemTime::now(); - let file_name = self.file_path.display().to_string(); - let reader = BufReader::new(File::open(file_name)?); + //let file_name = self.file_path.display().to_string(); + //let reader = BufReader::new(File::open(file_name)?); println!( "Reading file process expire mini seconds time: {:?}", SystemTime::now().duration_since(start_time)?.as_micros() ); start_time = SystemTime::now(); - let lines = reader.lines().collect::, _>>()?; + // let lines = reader.lines().collect::, _>>()?; + // let all_items = lines + // .par_iter() + // .map(|line| Ok(serde_json::from_str::(line.as_str()))?) + // .filter(|item| match item { + // Ok(i) => i.matched(&self.filter), + // Err(_e) => false, + // }) + // .collect::, _>>()?; if let Some(observer) = &self.obs { observer.before_progress()?; } - let all_items = lines - .par_iter() - .map(|line| Ok(serde_json::from_str::(line.as_str()))?) - .filter(|item| match item { - Ok(i) => { - if i.matched(&self.filter) { - true - } else { - false - } - } - Err(_e) => false, - }) - .collect::, _>>()?; - - let progress_bar = ProgressBar::new(all_items.len() as u64).with_style( + let progress_bar = ProgressBar::new(self.block_reader.get_progress_interval()).with_style( ProgressStyle::default_bar() .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), ); + let all_items = self.block_reader.read()?; + // .iter() + // .filter(|b| (*b).matched(&self.filter)) + // .map(|b| *b) + // .collect(); + println!( "Reading lines from file expire time: {:?}", SystemTime::now().duration_since(start_time)?.as_secs() @@ -194,8 +318,18 @@ impl ParallelCommandProgress { } } -pub trait ParallelCommand { +pub trait ParallelCommand { fn execute(&self, cmd: &CommandT) -> (usize, Vec); fn matched(&self, filter: &Option) -> bool; } + +pub trait ParallelCommandObserver { + fn before_progress(&self) -> Result<()>; + fn after_progress(&self) -> Result<()>; +} + +pub trait ParallelCommandBlockReader { + fn get_progress_interval(&self) -> u64; + fn read(&self) -> Result>; +} diff --git a/cmd/db-exporter/src/lib.rs b/cmd/db-exporter/src/lib.rs index a4d5215d21..4eef6f976d 100644 --- a/cmd/db-exporter/src/lib.rs +++ b/cmd/db-exporter/src/lib.rs @@ -1,30 +1,7 @@ // Copyright (c) The Starcoin Core Contributors // SPDX-License-Identifier: Apache-2.0 -use anyhow::Result; -use starcoin_storage::{ - cache_storage::CacheStorage, db_storage::DBStorage, storage::StorageInstance, Storage, - StorageVersion, -}; -use std::path::PathBuf; -use std::sync::Arc; - pub mod command_decode_payload; pub mod command_progress; pub mod verify_header; pub mod verify_module; - -pub fn init_db_obj(db_path: PathBuf) -> Result> { - let db_storage = DBStorage::open_with_cfs( - db_path.join("starcoindb/db/starcoindb"), - StorageVersion::current_version() - .get_column_family_names() - .to_vec(), - true, - Default::default(), - None, - )?; - Ok(Arc::new(Storage::new( - StorageInstance::new_cache_and_db_instance(CacheStorage::new(None), db_storage), - )?)) -} diff --git a/cmd/db-exporter/src/verify_header.rs b/cmd/db-exporter/src/verify_header.rs index 983c73e7f6..d8c8970f5b 100644 --- a/cmd/db-exporter/src/verify_header.rs +++ b/cmd/db-exporter/src/verify_header.rs @@ -4,8 +4,12 @@ use clap::Parser; use starcoin_consensus::{Consensus, G_CRYPTONIGHT}; use std::path::PathBuf; +use std::sync::Arc; -use crate::command_progress::{ParallelCommand, ParallelCommandFilter, ParallelCommandProgress}; +use crate::command_progress::{ + ParallelCommand, ParallelCommandFilter, ParallelCommandProgress, + ParallelCommandReadBodyFromExportLine, +}; use starcoin_types::block::Block; #[derive(Debug, Parser)] @@ -30,15 +34,15 @@ pub struct VerifyHeaderCmdType; pub fn verify_header_via_export_file(path: PathBuf, batch_size: usize) -> anyhow::Result<()> { let batch_cmd = ParallelCommandProgress::new( String::from("verify_block_header"), - path, batch_size, + Arc::new(ParallelCommandReadBodyFromExportLine::new(path)?), None, None, ); - batch_cmd.progress::(&VerifyHeaderCmdType {}) + batch_cmd.progress::(&VerifyHeaderCmdType {}) } -impl ParallelCommand for Block { +impl ParallelCommand for Block { fn execute(&self, _cmd: &VerifyHeaderCmdType) -> (usize, Vec) { let ret = G_CRYPTONIGHT.verify_header_difficulty(self.header.difficulty(), &self.header); match ret { diff --git a/cmd/db-exporter/src/verify_module.rs b/cmd/db-exporter/src/verify_module.rs index ed76ff0b89..20386bdde7 100644 --- a/cmd/db-exporter/src/verify_module.rs +++ b/cmd/db-exporter/src/verify_module.rs @@ -3,9 +3,13 @@ use move_binary_format::errors::Location; use starcoin_crypto::HashValue; use starcoin_types::{block::Block, transaction::TransactionPayload}; use starcoin_vm_types::{errors::VMError, file_format::CompiledModule}; +use std::sync::Arc; use std::{fmt::Debug, path::PathBuf}; //use starcoin_accumulator::node::AccumulatorStoreType::Block; -use crate::command_progress::{ParallelCommand, ParallelCommandFilter, ParallelCommandProgress}; +use crate::command_progress::{ + ParallelCommand, ParallelCommandFilter, ParallelCommandProgress, + ParallelCommandReadBodyFromExportLine, +}; #[derive(Debug, Parser)] #[clap( @@ -27,7 +31,7 @@ pub struct VerifyModuleError { pub struct VerifyModulesType; -impl ParallelCommand for Block { +impl ParallelCommand for Block { fn execute(&self, _cmd: &VerifyModulesType) -> (usize, Vec) { let mut errors = vec![]; let mut success_modules = 0; @@ -123,9 +127,14 @@ impl ParallelCommand for Block { // } pub fn verify_modules_via_export_file(input_path: PathBuf) -> anyhow::Result<()> { - let batch_cmd = - ParallelCommandProgress::new(String::from("verify_module"), input_path, 10, None, None); - batch_cmd.progress::(&VerifyModulesType {}) + let batch_cmd = ParallelCommandProgress::new( + String::from("verify_module"), + num_cpus::get(), + Arc::new(ParallelCommandReadBodyFromExportLine::new(input_path)?), + None, + None, + ); + batch_cmd.progress::(&VerifyModulesType {}) // let start_time = SystemTime::now(); // let file_name = input_path.display().to_string(); // let reader = BufReader::new(File::open(input_path)?); From 5cb6a56412e70af22ccf5957dec5d3cac34ec397 Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 14:36:45 +0800 Subject: [PATCH 05/16] fix timestamp error --- cmd/db-exporter/src/command_decode_payload.rs | 90 ++++++++++++------- cmd/db-exporter/src/command_progress.rs | 55 +++++++++--- 2 files changed, 101 insertions(+), 44 deletions(-) diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs index 57cd8d438b..1695c42231 100644 --- a/cmd/db-exporter/src/command_decode_payload.rs +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -6,14 +6,14 @@ use crate::command_progress::{ ParallelCommandReadBlockFromDB, }; use anyhow::Result; -use chrono::{TimeZone, Utc}; +use chrono::{DateTime, TimeZone, Utc}; use clap::Parser; use csv::{Writer, WriterBuilder}; use move_binary_format::errors::{Location, PartialVMError}; use serde::Serialize; use starcoin_abi_decoder; use starcoin_abi_decoder::DecodedTransactionPayload; -use starcoin_config::BuiltinNetworkID::Barnard; +use starcoin_config::BuiltinNetworkID::{Barnard, Main}; use starcoin_config::ChainNetwork; use starcoin_crypto::{hash::CryptoHash, HashValue}; use starcoin_statedb::ChainStateDB; @@ -22,6 +22,7 @@ use starcoin_types::{block::Block, transaction::TransactionPayload}; use starcoin_vm_types::errors::VMError; use std::fs::File; use std::sync::{Arc, Mutex}; +use std::time::{Duration, UNIX_EPOCH}; use std::{fmt::Debug, path::PathBuf}; const DECODE_PAYLOAD_COMMAND_NAME: &str = "decode_payload_command"; @@ -96,6 +97,7 @@ impl DecodePayloadCommandError { pub struct CSVHeaders { txn_hash: String, signer: String, + txn_type: String, func_name: String, ty_args: String, args: String, @@ -119,13 +121,21 @@ impl ParallelCommandObserver for CommandDecodePayload { } } +fn timestamp_to_datetime(timestamp: u64) -> String { + // Creates a new SystemTime from the specified number of whole seconds + let d = UNIX_EPOCH + Duration::from_secs(timestamp); + // Create DateTime from SystemTime + let datetime = DateTime::::from(d); + // Formats the combined date and time with the specified format string. + datetime.format("%Y-%m-%d %H:%M:%S.%f").to_string() +} + impl ParallelCommand for Block { fn execute(&self, command: &CommandDecodePayload) -> (usize, Vec) { // let errors = vec![]; // let mut success_module_size = 0; - let datetime = Utc.timestamp_opt(self.header.timestamp() as i64, 0); - let formatted_date = datetime.unwrap().format("%Y-%m-%d %H:%M:%s").to_string(); + let formatted_date = timestamp_to_datetime(self.header.timestamp() / 1000); let root = self.header.state_root(); let statedb = ChainStateDB::new(command.storage.clone(), Some(root)); @@ -136,31 +146,51 @@ impl ParallelCommand for Block starcoin_abi_decoder::decode_txn_payload(&statedb, txn.payload()) .expect("Decode transaction payload failed!"); + let mut writer = command.writer_mutex.lock().unwrap(); match decoded_txn_payload { - DecodedTransactionPayload::ScriptFunction(payload) => { - let mut writer = command.writer_mutex.lock().unwrap(); - writer - .serialize(CSVHeaders { - txn_hash: txn.hash().to_string(), - signer, - func_name: format!("{}::{}", payload.module, payload.function), - ty_args: payload - .ty_args - .iter() - .map(|a| a.to_string()) - .collect::>() - .join(","), - args: payload - .args - .iter() - .map(|a| a.0.to_string()) - .collect::>() - .join(","), - timestamp: formatted_date.clone(), - }) - .expect("Write into CSV failed!") - } - DecodedTransactionPayload::Script(_) | DecodedTransactionPayload::Package(_) => (), + DecodedTransactionPayload::ScriptFunction(payload) => writer + .serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + txn_type: String::from("ScriptFunction"), + signer, + func_name: format!("{}::{}", payload.module, payload.function), + ty_args: payload + .ty_args + .iter() + .map(|a| a.to_string()) + .collect::>() + .join("|"), + args: payload + .args + .iter() + .map(|a| a.0.to_string()) + .collect::>() + .join("|"), + timestamp: formatted_date.clone(), + }) + .expect("Write into CSV failed!"), + DecodedTransactionPayload::Script(script) => writer + .serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + txn_type: String::from("Script"), + signer, + func_name: "".to_string(), + ty_args: "".to_string(), + args: "".to_string(), + timestamp: formatted_date.clone(), + }) + .expect("Write into CSV failed!"), + DecodedTransactionPayload::Package(package) => writer + .serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + txn_type: String::from("Package"), + signer, + func_name: "".to_string(), + ty_args: "".to_string(), + args: "".to_string(), + timestamp: formatted_date.clone(), + }) + .expect("Write into CSV failed!"), } } //(success_module_size, errors) @@ -201,7 +231,7 @@ pub fn decode_payload( let (dbreader, storage) = ParallelCommandReadBlockFromDB::new( input_path, - ChainNetwork::from(Barnard), + ChainNetwork::from(Main), start_height.unwrap_or(0), end_height.unwrap_or(0), )?; @@ -223,7 +253,7 @@ pub fn decode_payload( #[test] pub fn test_decode_payload() -> Result<()> { decode_payload( - PathBuf::from("~/.starcoin/barnard"), + PathBuf::from("/Users/bobong/.starcoin/main"), PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet/output.csv"), None, None, diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 00035e65fe..84fd59a900 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -10,9 +10,14 @@ use starcoin_storage::storage::StorageInstance; use starcoin_storage::{Storage, StorageVersion}; use starcoin_types::block::{Block, BlockNumber}; use starcoin_vm_types::language_storage::TypeTag; -use std::sync::Arc; -use std::{fs::File, io::{BufRead, BufReader}, path::PathBuf, time::SystemTime}; use std::io::{Seek, SeekFrom}; +use std::sync::Arc; +use std::{ + fs::File, + io::{BufRead, BufReader}, + path::PathBuf, + time::SystemTime, +}; struct CommandResult { succeed: usize, @@ -78,7 +83,6 @@ pub struct ParallelCommandReadBodyFromExportLine { } impl ParallelCommandReadBodyFromExportLine { - fn count_lines(reader: &mut BufReader) -> Result { let line_count = reader.lines().count(); reader.seek(SeekFrom::Start(0))?; @@ -87,11 +91,10 @@ impl ParallelCommandReadBodyFromExportLine { pub fn new(input_path: PathBuf) -> Result { let file = File::open(input_path.display().to_string())?; - let line_count = ParallelCommandReadBodyFromExportLine::count_lines(&mut BufReader::new(file.try_clone()?))?; - Ok(Self { - file, - line_count, - }) + let line_count = ParallelCommandReadBodyFromExportLine::count_lines(&mut BufReader::new( + file.try_clone()?, + ))?; + Ok(Self { file, line_count }) } } @@ -129,22 +132,27 @@ impl ParallelCommandReadBlockFromDB { let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), input_path.as_ref()) .expect("Failed init_and_check_storage"); - let chain = BlockChain::new(net.time_service(), chain_info.head().id(), storage.clone(), None) - .expect("Failed to initialize block chain"); + let chain = BlockChain::new( + net.time_service(), + chain_info.head().id(), + storage.clone(), + None, + ) + .expect("Failed to initialize block chain"); let cur_num = chain.status().head().number(); - let (start_num, end_num) = if start != 0 && end == 0 { + let (start_num, end_num) = if start == 0 && end == 0 { (0, cur_num) } else { - let end = if cur_num > end + BLOCK_GAP { + let final_end = if cur_num > end + BLOCK_GAP { end } else if cur_num > BLOCK_GAP { cur_num - BLOCK_GAP } else { end }; - (start, end) + (start, final_end) }; if start > cur_num || start > end { @@ -291,7 +299,6 @@ impl ParallelCommandProgress { .collect::>() }) .collect::>>(); - let result = excution_result.into_iter().flatten().fold( CommandResult { succeed: 0, @@ -302,6 +309,26 @@ impl ParallelCommandProgress { failed: acc.failed + result.failed, }, ); + // + // let excution_result = all_items + // .iter() + // .map(|item| { + // let (succeed, failed) = item.execute(command); + // progress_bar.inc(1); + // CommandResult::new(succeed, failed.len()) + // }) + // .collect::>(); + // + // let result = excution_result.into_iter().fold( + // CommandResult { + // succeed: 0, + // failed: 0, + // }, + // |acc, result| CommandResult { + // succeed: acc.succeed + result.succeed, + // failed: acc.failed + result.failed, + // }, + // ); progress_bar.finish(); From 4214458d436c10fd49b4e188295640ef21ed3004 Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 16:05:39 +0800 Subject: [PATCH 06/16] add decode payload to main command --- cmd/db-exporter/src/command_decode_payload.rs | 125 +++++++++++++----- cmd/db-exporter/src/command_progress.rs | 88 ++++++------ cmd/db-exporter/src/main.rs | 17 ++- 3 files changed, 145 insertions(+), 85 deletions(-) diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs index 1695c42231..7dca664a18 100644 --- a/cmd/db-exporter/src/command_decode_payload.rs +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -13,12 +13,12 @@ use move_binary_format::errors::{Location, PartialVMError}; use serde::Serialize; use starcoin_abi_decoder; use starcoin_abi_decoder::DecodedTransactionPayload; -use starcoin_config::BuiltinNetworkID::{Barnard, Main}; -use starcoin_config::ChainNetwork; +use starcoin_config::BuiltinNetworkID::Main; +use starcoin_config::{BuiltinNetworkID, ChainNetwork}; use starcoin_crypto::{hash::CryptoHash, HashValue}; use starcoin_statedb::ChainStateDB; use starcoin_storage::Storage; -use starcoin_types::{block::Block, transaction::TransactionPayload}; +use starcoin_types::{block::Block, block::BlockNumber, transaction::TransactionPayload}; use starcoin_vm_types::errors::VMError; use std::fs::File; use std::sync::{Arc, Mutex}; @@ -33,21 +33,25 @@ const DECODE_PAYLOAD_COMMAND_NAME: &str = "decode_payload_command"; about = "Decode payload for given parameter and function name" )] pub struct DecodePayloadCommandOptions { - #[clap(long, short = 'i', parse(from_os_str))] - /// Db path - pub input_path: PathBuf, + #[clap(long, short = 'n')] + /// Chain Network, like main, proxima + pub net: BuiltinNetworkID, #[clap(long, short = 'o', parse(from_os_str))] - /// output file, like accounts.csv - pub output_path: PathBuf, - - #[clap(long)] - pub start_height: Option, + /// output dir, like ~/, output filename like ~/block_start_end.csv + pub output: PathBuf, - #[clap(long)] - pub end_height: Option, + #[clap(long, short = 'i', parse(from_os_str))] + /// starcoin node db path. like ~/.starcoin/main + pub db_path: PathBuf, #[clap(long, short = 's')] + pub start: Option, + + #[clap(long, short = 'e')] + pub end: Option, + + #[clap(long, short = 'c')] /// Signer filter pub signer: Option, @@ -95,13 +99,15 @@ impl DecodePayloadCommandError { #[derive(Serialize)] pub struct CSVHeaders { + block_num: String, txn_hash: String, signer: String, txn_type: String, func_name: String, ty_args: String, args: String, - timestamp: String, + timestamp: u64, + date_time: String, } pub struct CommandDecodePayload { @@ -135,10 +141,12 @@ impl ParallelCommand for Block // let errors = vec![]; // let mut success_module_size = 0; - let formatted_date = timestamp_to_datetime(self.header.timestamp() / 1000); + let timestamp = self.header.timestamp() / 1000; + let formatted_date = timestamp_to_datetime(timestamp); let root = self.header.state_root(); let statedb = ChainStateDB::new(command.storage.clone(), Some(root)); + let block_num = self.header.number().to_string(); for txn in self.transactions() { let signer = txn.sender().to_string(); @@ -150,6 +158,7 @@ impl ParallelCommand for Block match decoded_txn_payload { DecodedTransactionPayload::ScriptFunction(payload) => writer .serialize(CSVHeaders { + block_num: block_num.clone(), txn_hash: txn.hash().to_string(), txn_type: String::from("ScriptFunction"), signer, @@ -166,29 +175,34 @@ impl ParallelCommand for Block .map(|a| a.0.to_string()) .collect::>() .join("|"), - timestamp: formatted_date.clone(), + timestamp, + date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), DecodedTransactionPayload::Script(script) => writer .serialize(CSVHeaders { + block_num: block_num.clone(), txn_hash: txn.hash().to_string(), txn_type: String::from("Script"), signer, func_name: "".to_string(), ty_args: "".to_string(), args: "".to_string(), - timestamp: formatted_date.clone(), + timestamp, + date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), DecodedTransactionPayload::Package(package) => writer .serialize(CSVHeaders { + block_num: block_num.clone(), txn_hash: txn.hash().to_string(), txn_type: String::from("Package"), signer, func_name: "".to_string(), ty_args: "".to_string(), args: "".to_string(), - timestamp: formatted_date.clone(), + timestamp, + date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), } @@ -202,24 +216,46 @@ impl ParallelCommand for Block /// and return false if any condition is met. /// fn matched(&self, filters: &Option) -> bool { - filters.as_ref().map_or_else( - || true, - |f| { + if self.transactions().is_empty() { + return true; + }; + + match filters { + Some(filter) => { + println!("has key"); self.transactions().iter().any(|txn| match txn.payload() { TransactionPayload::ScriptFunction(payload) => { - f.match_signer(&txn.sender().to_string()) - || f.match_func_name(payload.function().as_str()) - || f.match_ty_args(&payload.ty_args().to_vec()) - || f.match_args(&payload.args().to_vec()) + filter.match_signer(&txn.sender().to_string()) + || filter.match_func_name(payload.function().as_str()) + || filter.match_ty_args(&payload.ty_args().to_vec()) + || filter.match_args(&payload.args().to_vec()) } _ => true, }) }, - ) + None => true, + } } } -pub fn decode_payload( +pub fn do_decode_payload_command(option: &DecodePayloadCommandOptions) -> Result<()> { + do_decode_payload( + option.net, + option.db_path.clone(), + option.output.clone(), + option.start, + option.end, + ParallelCommandFilter::new( + &option.signer, + &option.func_name, + &option.ty_args, + &option.args, + ), + ) +} + +pub fn do_decode_payload( + net: BuiltinNetworkID, input_path: PathBuf, out_path: PathBuf, start_height: Option, @@ -231,9 +267,10 @@ pub fn decode_payload( let (dbreader, storage) = ParallelCommandReadBlockFromDB::new( input_path, - ChainNetwork::from(Main), + ChainNetwork::from(net), start_height.unwrap_or(0), end_height.unwrap_or(0), + true, )?; let command = Arc::new(CommandDecodePayload { writer_mutex, @@ -252,11 +289,33 @@ pub fn decode_payload( #[test] pub fn test_decode_payload() -> Result<()> { - decode_payload( - PathBuf::from("/Users/bobong/.starcoin/main"), - PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet/output.csv"), - None, + let input = PathBuf::from("/Users/bobong/.starcoin/main"); + let output = PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet/output.csv"); + + // do_decode_payload(Main, input.clone(), output.clone(), None, None, None)?; + + // do_decode_payload( + // Main, + // input.clone(), + // output.clone(), + // Some(0), + // Some(100), + // None, + // )?; + // + do_decode_payload( + Main, + input.clone(), + output.clone(), None, None, - ) + ParallelCommandFilter::new( + &Some("0x45b467f509bb82f8bcbd7b01170a22d0".to_string()), + &None, + &None, + &None, + ), + )?; + + Ok(()) } diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 84fd59a900..631f3ac0fe 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -40,18 +40,18 @@ pub struct ParallelCommandFilter { } impl ParallelCommandFilter { - fn new( - signer: Option, - func_name: Option, - ty_args: Option>, - args: Option>, + pub fn new( + signer: &Option, + func_name: &Option, + ty_args: &Option>, + args: &Option>, ) -> Option { if func_name.is_some() || ty_args.is_some() || args.is_some() { Some(ParallelCommandFilter { - signer, - func_name, - ty_args, - args, + signer: signer.clone(), + func_name: func_name.clone(), + ty_args: ty_args.clone(), + args: args.clone(), }) } else { None @@ -117,6 +117,7 @@ pub struct ParallelCommandReadBlockFromDB { start_num: u64, end_num: u64, chain: Arc, + skip_empty_block: bool, } const BLOCK_GAP: u64 = 1000; @@ -127,6 +128,7 @@ impl ParallelCommandReadBlockFromDB { net: ChainNetwork, start: u64, end: u64, + skip_empty_block: bool, ) -> Result<(Self, Arc)> { let storage = Self::init_db_obj(input_path.clone()).expect("Failed to initialize db"); let (chain_info, _) = @@ -169,6 +171,7 @@ impl ParallelCommandReadBlockFromDB { start_num, end_num, chain: Arc::new(chain), + skip_empty_block, }, storage, )) @@ -195,18 +198,26 @@ impl ParallelCommandBlockReader for ParallelCommandReadBlockFromDB { } fn read(&self) -> Result> { - let ret = (self.start_num..=self.end_num) + println!( + "ParallelCommandBlockReader::read | read range: {}, {}, skip empty block: {}", + self.start_num, self.end_num, self.skip_empty_block + ); + + Ok((self.start_num..=self.end_num) .collect::>() .into_iter() - .map(|num| { - // progress_bar.set_message(format!("load block {}", num)); - // progress_bar.inc(1); - self.chain.get_block_by_number(num).ok()? + .filter_map(|num| { + if self.skip_empty_block { + self.chain + .get_block_by_number(num) + .ok() + .flatten() + .filter(|block| !block.transactions().is_empty()) + } else { + self.chain.get_block_by_number(num).ok().flatten() + } }) - .filter(|block| block.is_some()) - .map(|block| block.unwrap()) - .collect(); - Ok(ret) + .collect::>()) } } @@ -264,20 +275,24 @@ impl ParallelCommandProgress { observer.before_progress()?; } + let progress_interval = self.block_reader.get_progress_interval(); let progress_bar = ProgressBar::new(self.block_reader.get_progress_interval()).with_style( ProgressStyle::default_bar() .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), ); - let all_items = self.block_reader.read()?; - // .iter() - // .filter(|b| (*b).matched(&self.filter)) - // .map(|b| *b) - // .collect(); + let mut all_items = self.block_reader.read()?; + let all_item_size = all_items.len(); + + all_items.retain(|b| (*b).matched(&self.filter)); + let filtered_item_size = all_items.len(); println!( - "Reading lines from file expire time: {:?}", - SystemTime::now().duration_since(start_time)?.as_secs() + "Reading lines from file expire time: {:?}, get interval: {:?}, actual return item size: {:?}, filtered item size:{:?}", + SystemTime::now().duration_since(start_time)?.as_secs(), + progress_interval, + all_item_size, + filtered_item_size, ); // It is necessary to divide all rows into subsets @@ -309,30 +324,9 @@ impl ParallelCommandProgress { failed: acc.failed + result.failed, }, ); - // - // let excution_result = all_items - // .iter() - // .map(|item| { - // let (succeed, failed) = item.execute(command); - // progress_bar.inc(1); - // CommandResult::new(succeed, failed.len()) - // }) - // .collect::>(); - // - // let result = excution_result.into_iter().fold( - // CommandResult { - // succeed: 0, - // failed: 0, - // }, - // |acc, result| CommandResult { - // succeed: acc.succeed + result.succeed, - // failed: acc.failed + result.failed, - // }, - // ); - progress_bar.finish(); - println!("verify {:?}, use time: {:?}, success modules: {}, error modules: {}, total modules: {}", + println!("Running ParallelCommand {:?}, use time: {:?}, success modules: {}, error modules: {}, total modules: {}", self.name, SystemTime::now().duration_since(start_time)?.as_secs(), result.succeed, result.failed, result.succeed + result.failed); if result.failed > 0 { bail!("verify block modules error"); diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs index 98b9d02f2c..f8d83ce1f7 100644 --- a/cmd/db-exporter/src/main.rs +++ b/cmd/db-exporter/src/main.rs @@ -5,7 +5,9 @@ use anyhow::{bail, format_err, Result}; use bcs_ext::{BCSCodec, Sample}; use clap::{IntoApp, Parser}; use csv::Writer; +use db_exporter::command_decode_payload::do_decode_payload_command; use db_exporter::{ + command_decode_payload::{do_decode_payload, DecodePayloadCommandOptions}, verify_header::{verify_header_via_export_file, VerifyHeaderOptions}, verify_module::{verify_modules_via_export_file, VerifyModuleOptions}, }; @@ -235,6 +237,7 @@ enum Cmd { VerifyHeader(VerifyHeaderOptions), GenTurboSTMTransactions(GenTurboSTMTransactionsOptions), ApplyTurboSTMBlock(ApplyTurboSTMBlockOptions), + DecodePayload(DecodePayloadCommandOptions), } #[derive(Debug, Clone, Parser)] @@ -259,7 +262,7 @@ pub struct CheckKeyOptions { /// starcoin node db path. like ~/.starcoin/barnard/starcoindb/db/starcoindb pub db_path: PathBuf, #[clap(long, short = 'n', - possible_values=&["block", "block_header"],)] + possible_values = & ["block", "block_header"],)] pub cf_name: String, #[clap(long, short = 'b')] pub block_hash: HashValue, @@ -350,7 +353,7 @@ pub struct GenBlockTransactionsOptions { pub block_num: Option, #[clap(long, short = 't')] pub trans_num: Option, - #[clap(long, short = 'p', possible_values=&["CreateAccount", "FixAccount", "EmptyTxn"],)] + #[clap(long, short = 'p', possible_values = & ["CreateAccount", "FixAccount", "EmptyTxn"],)] /// txn type pub txn_type: Txntype, } @@ -404,9 +407,9 @@ pub struct ExportResourceOptions { pub block_hash: HashValue, #[clap( - short='r', - default_value = "0x1::Account::Balance<0x1::STC::STC>", - parse(try_from_str=parse_struct_tag) + short = 'r', + default_value = "0x1::Account::Balance<0x1::STC::STC>", + parse(try_from_str = parse_struct_tag) )] /// resource struct tag. resource_type: StructTag, @@ -607,6 +610,9 @@ async fn main() -> anyhow::Result<()> { apply_turbo_stm_block(option.to_path, option.turbo_stm_to_path, option.input_path); return result; } + Cmd::DecodePayload(option) => { + return do_decode_payload_command(&option); + } } Ok(()) } @@ -856,6 +862,7 @@ pub fn gen_block_transactions( } } } + /// Returns a transaction to create a new account with the given arguments. pub fn create_account_txn_sent_as_association( new_account: &Account, From 53e59311fcd64488ba9fa75960852a4e59157ed1 Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 16:19:52 +0800 Subject: [PATCH 07/16] commit all code --- Cargo.lock | 2 ++ cmd/db-exporter/src/main.rs | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2e0c1897d..83951801a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2117,6 +2117,7 @@ dependencies = [ "anyhow", "atomic-counter", "bcs-ext", + "chrono", "clap 3.2.23", "csv", "futures 0.3.26", @@ -2129,6 +2130,7 @@ dependencies = [ "rayon", "serde 1.0.152", "serde_json", + "starcoin-abi-decoder", "starcoin-account-api", "starcoin-accumulator", "starcoin-chain", diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs index f8d83ce1f7..f1330b3707 100644 --- a/cmd/db-exporter/src/main.rs +++ b/cmd/db-exporter/src/main.rs @@ -5,9 +5,8 @@ use anyhow::{bail, format_err, Result}; use bcs_ext::{BCSCodec, Sample}; use clap::{IntoApp, Parser}; use csv::Writer; -use db_exporter::command_decode_payload::do_decode_payload_command; use db_exporter::{ - command_decode_payload::{do_decode_payload, DecodePayloadCommandOptions}, + command_decode_payload::{do_decode_payload_command, DecodePayloadCommandOptions}, verify_header::{verify_header_via_export_file, VerifyHeaderOptions}, verify_module::{verify_modules_via_export_file, VerifyModuleOptions}, }; From e9203442eaf61886cf7bf2b90cab6a66d1a659ab Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 17:05:26 +0800 Subject: [PATCH 08/16] add signer filter --- cmd/db-exporter/src/command_decode_payload.rs | 20 +++++++++++-------- cmd/db-exporter/src/command_progress.rs | 12 ++++++++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs index 7dca664a18..5e9301a549 100644 --- a/cmd/db-exporter/src/command_decode_payload.rs +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -222,15 +222,19 @@ impl ParallelCommand for Block match filters { Some(filter) => { - println!("has key"); - self.transactions().iter().any(|txn| match txn.payload() { - TransactionPayload::ScriptFunction(payload) => { - filter.match_signer(&txn.sender().to_string()) - || filter.match_func_name(payload.function().as_str()) - || filter.match_ty_args(&payload.ty_args().to_vec()) - || filter.match_args(&payload.args().to_vec()) + self.transactions().iter().any(|txn| { + match txn.payload() { + TransactionPayload::ScriptFunction(payload) => { + filter.match_signer(&txn.sender().to_string()) + && filter.match_func_name(payload.function().as_str()) + && filter.match_ty_args(&payload.ty_args().to_vec()) + && filter.match_args(&payload.args().to_vec()) + }, + TransactionPayload::Script(_) | TransactionPayload::Package(_) => { + filter.match_signer(&txn.sender().to_string()) + }, + _ => true, } - _ => true, }) }, None => true, diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 631f3ac0fe..4fe48585b5 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -46,7 +46,7 @@ impl ParallelCommandFilter { ty_args: &Option>, args: &Option>, ) -> Option { - if func_name.is_some() || ty_args.is_some() || args.is_some() { + if signer.is_some() || func_name.is_some() || ty_args.is_some() || args.is_some() { Some(ParallelCommandFilter { signer: signer.clone(), func_name: func_name.clone(), @@ -59,11 +59,17 @@ impl ParallelCommandFilter { } pub fn match_signer(&self, signer: &str) -> bool { - self.signer.as_ref().map_or(false, |n| n == signer) + if self.signer.is_none() { + return true; + } + (self.signer.clone().unwrap() == signer) } pub fn match_func_name(&self, func_name: &str) -> bool { - self.func_name.as_ref().map_or(false, |n| n == func_name) + if self.func_name.is_none() { + return true; + } + (self.func_name.clone().unwrap() == func_name) } pub fn match_ty_args(&self, _ty_args: &Vec) -> bool { From 7aaae2a1eeae4c7e9b3ebce7371a2fffdb28bd93 Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 20:22:07 +0800 Subject: [PATCH 09/16] change the loading block action to parallel --- cmd/db-exporter/src/command_progress.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 4fe48585b5..1de301510a 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -211,16 +211,17 @@ impl ParallelCommandBlockReader for ParallelCommandReadBlockFromDB { Ok((self.start_num..=self.end_num) .collect::>() - .into_iter() + .par_iter() .filter_map(|num| { + println!("Read block number: {}", num); if self.skip_empty_block { self.chain - .get_block_by_number(num) + .get_block_by_number(*num) .ok() .flatten() .filter(|block| !block.transactions().is_empty()) } else { - self.chain.get_block_by_number(num).ok().flatten() + self.chain.get_block_by_number(*num).ok().flatten() } }) .collect::>()) From 9679d68be1fed53dd94c809e125b3db0e66c86eb Mon Sep 17 00:00:00 2001 From: welbon Date: Tue, 20 Feb 2024 21:12:46 +0800 Subject: [PATCH 10/16] fix progress bar show --- cmd/db-exporter/src/command_progress.rs | 46 +++++++++++++------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 1de301510a..73cedcce27 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -109,12 +109,16 @@ impl ParallelCommandBlockReader for ParallelCommandReadBodyFromExportLine { self.line_count } - fn read(&self) -> Result> { + fn read(&self, load_bar: &ProgressBar) -> Result> { let reader = BufReader::new(self.file.try_clone()?); let lines = reader.lines().collect::, _>>()?; Ok(lines .par_iter() - .map(|line| Ok(serde_json::from_str::(line.as_str()))?) + .map(|line| { + let ret = serde_json::from_str::(line.as_str()); + load_bar.inc(1); + Ok(ret)? + }) .collect::, _>>()?) } } @@ -203,7 +207,7 @@ impl ParallelCommandBlockReader for ParallelCommandReadBlockFromDB { self.end_num - self.start_num } - fn read(&self) -> Result> { + fn read(&self, load_bar: &ProgressBar) -> Result> { println!( "ParallelCommandBlockReader::read | read range: {}, {}, skip empty block: {}", self.start_num, self.end_num, self.skip_empty_block @@ -213,7 +217,7 @@ impl ParallelCommandBlockReader for ParallelCommandReadBlockFromDB { .collect::>() .par_iter() .filter_map(|num| { - println!("Read block number: {}", num); + load_bar.inc(1); if self.skip_empty_block { self.chain .get_block_by_number(*num) @@ -224,7 +228,8 @@ impl ParallelCommandBlockReader for ParallelCommandReadBlockFromDB { self.chain.get_block_by_number(*num).ok().flatten() } }) - .collect::>()) + .collect::>() + ) } } @@ -268,36 +273,28 @@ impl ParallelCommandProgress { ); start_time = SystemTime::now(); - // let lines = reader.lines().collect::, _>>()?; - - // let all_items = lines - // .par_iter() - // .map(|line| Ok(serde_json::from_str::(line.as_str()))?) - // .filter(|item| match item { - // Ok(i) => i.matched(&self.filter), - // Err(_e) => false, - // }) - // .collect::, _>>()?; + if let Some(observer) = &self.obs { observer.before_progress()?; } - let progress_interval = self.block_reader.get_progress_interval(); - let progress_bar = ProgressBar::new(self.block_reader.get_progress_interval()).with_style( + let load_interval_count = self.block_reader.get_progress_interval(); + let load_bar = ProgressBar::new(load_interval_count).with_style( ProgressStyle::default_bar() - .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), + .template("loading [{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), ); - let mut all_items = self.block_reader.read()?; - let all_item_size = all_items.len(); + let mut all_items = self.block_reader.read(&load_bar)?; + load_bar.finish(); + let all_item_size = all_items.len(); all_items.retain(|b| (*b).matched(&self.filter)); let filtered_item_size = all_items.len(); println!( "Reading lines from file expire time: {:?}, get interval: {:?}, actual return item size: {:?}, filtered item size:{:?}", SystemTime::now().duration_since(start_time)?.as_secs(), - progress_interval, + load_interval_count, all_item_size, filtered_item_size, ); @@ -307,6 +304,10 @@ impl ParallelCommandProgress { // so that they can be divided into several threads for the following operations start_time = SystemTime::now(); + let progress_bar = ProgressBar::new(all_items.len() as u64).with_style( + ProgressStyle::default_bar() + .template("processing [{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), + ); let excution_result = all_items .into_par_iter() .chunks(self.parallel_level) @@ -321,6 +322,7 @@ impl ParallelCommandProgress { .collect::>() }) .collect::>>(); + let result = excution_result.into_iter().flatten().fold( CommandResult { succeed: 0, @@ -359,5 +361,5 @@ pub trait ParallelCommandObserver { pub trait ParallelCommandBlockReader { fn get_progress_interval(&self) -> u64; - fn read(&self) -> Result>; + fn read(&self, load_bar: &ProgressBar) -> Result>; } From 47cb15909af211eabe71feef1efdff7ea92f4f2b Mon Sep 17 00:00:00 2001 From: welbon Date: Wed, 21 Feb 2024 12:11:11 +0800 Subject: [PATCH 11/16] fixed clippy --- cmd/db-exporter/src/command_decode_payload.rs | 63 +++++++++---------- cmd/db-exporter/src/command_progress.rs | 18 ++++-- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs index 5e9301a549..54692d50df 100644 --- a/cmd/db-exporter/src/command_decode_payload.rs +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -6,14 +6,12 @@ use crate::command_progress::{ ParallelCommandReadBlockFromDB, }; use anyhow::Result; -use chrono::{DateTime, TimeZone, Utc}; +use chrono::{DateTime, Utc}; use clap::Parser; use csv::{Writer, WriterBuilder}; -use move_binary_format::errors::{Location, PartialVMError}; use serde::Serialize; use starcoin_abi_decoder; use starcoin_abi_decoder::DecodedTransactionPayload; -use starcoin_config::BuiltinNetworkID::Main; use starcoin_config::{BuiltinNetworkID, ChainNetwork}; use starcoin_crypto::{hash::CryptoHash, HashValue}; use starcoin_statedb::ChainStateDB; @@ -75,27 +73,27 @@ pub struct DecodePayloadCommandError { pub error: VMError, } -impl DecodePayloadCommandError { - fn new_from_vm_error(error: VMError, block_number: u64, txn_hash: &HashValue) -> Self { - DecodePayloadCommandError { - block_number, - txn_hash: txn_hash.clone(), - error, - } - } - - fn new_from_partial_vm_error( - error: PartialVMError, - block_number: u64, - txn_hash: &HashValue, - ) -> Self { - DecodePayloadCommandError { - block_number, - txn_hash: txn_hash.clone(), - error: error.finish(Location::Undefined), - } - } -} +// impl DecodePayloadCommandError { +// fn new_from_vm_error(error: VMError, block_number: u64, txn_hash: &HashValue) -> Self { +// DecodePayloadCommandError { +// block_number, +// txn_hash: txn_hash.clone(), +// error, +// } +// } +// +// fn new_from_partial_vm_error( +// error: PartialVMError, +// block_number: u64, +// txn_hash: &HashValue, +// ) -> Self { +// DecodePayloadCommandError { +// block_number, +// txn_hash: txn_hash.clone(), +// error: error.finish(Location::Undefined), +// } +// } +// } #[derive(Serialize)] pub struct CSVHeaders { @@ -179,7 +177,7 @@ impl ParallelCommand for Block date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), - DecodedTransactionPayload::Script(script) => writer + DecodedTransactionPayload::Script(_) => writer .serialize(CSVHeaders { block_num: block_num.clone(), txn_hash: txn.hash().to_string(), @@ -192,7 +190,7 @@ impl ParallelCommand for Block date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), - DecodedTransactionPayload::Package(package) => writer + DecodedTransactionPayload::Package(_) => writer .serialize(CSVHeaders { block_num: block_num.clone(), txn_hash: txn.hash().to_string(), @@ -227,13 +225,12 @@ impl ParallelCommand for Block TransactionPayload::ScriptFunction(payload) => { filter.match_signer(&txn.sender().to_string()) && filter.match_func_name(payload.function().as_str()) - && filter.match_ty_args(&payload.ty_args().to_vec()) - && filter.match_args(&payload.args().to_vec()) + && filter.match_ty_args(payload.ty_args()) + && filter.match_args(payload.args()) }, TransactionPayload::Script(_) | TransactionPayload::Package(_) => { filter.match_signer(&txn.sender().to_string()) }, - _ => true, } }) }, @@ -266,7 +263,7 @@ pub fn do_decode_payload( end_height: Option, filter: Option, ) -> Result<()> { - let file = WriterBuilder::new().from_path(out_path.clone())?; + let file = WriterBuilder::new().from_path(out_path)?; let writer_mutex = Mutex::new(file); let (dbreader, storage) = ParallelCommandReadBlockFromDB::new( @@ -308,9 +305,9 @@ pub fn test_decode_payload() -> Result<()> { // )?; // do_decode_payload( - Main, - input.clone(), - output.clone(), + BuiltinNetworkID::Main, + input, + output, None, None, ParallelCommandFilter::new( diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 73cedcce27..50aabae902 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -62,23 +62,29 @@ impl ParallelCommandFilter { if self.signer.is_none() { return true; } - (self.signer.clone().unwrap() == signer) + self.signer.clone().unwrap() == signer } pub fn match_func_name(&self, func_name: &str) -> bool { if self.func_name.is_none() { return true; } - (self.func_name.clone().unwrap() == func_name) + self.func_name.clone().unwrap() == func_name } - pub fn match_ty_args(&self, _ty_args: &Vec) -> bool { - // TODO(Bob): To Compare + pub fn match_ty_args(&self, _ty_args: &[TypeTag]) -> bool { + if self.ty_args.is_some() { + print!("match_ty_args | {:?}", self.ty_args); + // TODO(Bob): To Compare + } true } - pub fn match_args(&self, _args: &Vec>) -> bool { - // TODO(Bob): To Compare + pub fn match_args(&self, _args: &[Vec]) -> bool { + if self.args.is_some() { + print!("match_args | {:?}", self.args); + // TODO(Bob): To Compare + } true } } From e720db30fc0c258a0b842da38b3dddc0b1cc0004 Mon Sep 17 00:00:00 2001 From: welbon Date: Thu, 22 Feb 2024 08:53:52 +0800 Subject: [PATCH 12/16] reformat command code --- .../src/{command_decode_payload.rs => commands/decode_payload.rs} | 0 cmd/db-exporter/src/commands/mod.rs | 0 cmd/db-exporter/src/{ => commands}/verify_header.rs | 0 cmd/db-exporter/src/{ => commands}/verify_module.rs | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename cmd/db-exporter/src/{command_decode_payload.rs => commands/decode_payload.rs} (100%) create mode 100644 cmd/db-exporter/src/commands/mod.rs rename cmd/db-exporter/src/{ => commands}/verify_header.rs (100%) rename cmd/db-exporter/src/{ => commands}/verify_module.rs (100%) diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/commands/decode_payload.rs similarity index 100% rename from cmd/db-exporter/src/command_decode_payload.rs rename to cmd/db-exporter/src/commands/decode_payload.rs diff --git a/cmd/db-exporter/src/commands/mod.rs b/cmd/db-exporter/src/commands/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/db-exporter/src/verify_header.rs b/cmd/db-exporter/src/commands/verify_header.rs similarity index 100% rename from cmd/db-exporter/src/verify_header.rs rename to cmd/db-exporter/src/commands/verify_header.rs diff --git a/cmd/db-exporter/src/verify_module.rs b/cmd/db-exporter/src/commands/verify_module.rs similarity index 100% rename from cmd/db-exporter/src/verify_module.rs rename to cmd/db-exporter/src/commands/verify_module.rs From bb5f1567c9ca5f9195d5e23b824bc41f7237de09 Mon Sep 17 00:00:00 2001 From: welbon Date: Thu, 22 Feb 2024 15:26:04 +0800 Subject: [PATCH 13/16] add execute state for transaction payload export --- cmd/db-exporter/src/command_progress.rs | 34 ++++++++-- .../src/commands/decode_payload.rs | 66 ++++++++++++------- cmd/db-exporter/src/commands/mod.rs | 3 + cmd/db-exporter/src/commands/verify_header.rs | 8 ++- cmd/db-exporter/src/commands/verify_module.rs | 15 +++-- cmd/db-exporter/src/lib.rs | 4 +- cmd/db-exporter/src/main.rs | 17 +++-- 7 files changed, 101 insertions(+), 46 deletions(-) diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 50aabae902..558d462cae 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -3,6 +3,7 @@ use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::*; use starcoin_chain::{BlockChain, ChainReader}; use starcoin_config::ChainNetwork; +use starcoin_crypto::HashValue; use starcoin_genesis::Genesis; use starcoin_storage::cache_storage::CacheStorage; use starcoin_storage::db_storage::DBStorage; @@ -10,6 +11,7 @@ use starcoin_storage::storage::StorageInstance; use starcoin_storage::{Storage, StorageVersion}; use starcoin_types::block::{Block, BlockNumber}; use starcoin_vm_types::language_storage::TypeTag; +use starcoin_vm_types::vm_status::KeptVMStatus; use std::io::{Seek, SeekFrom}; use std::sync::Arc; use std::{ @@ -127,6 +129,10 @@ impl ParallelCommandBlockReader for ParallelCommandReadBodyFromExportLine { }) .collect::, _>>()?) } + + fn query_txn_exec_state(&self, _txn_hash: HashValue) -> String { + "OK".to_string() + } } pub struct ParallelCommandReadBlockFromDB { @@ -234,8 +240,21 @@ impl ParallelCommandBlockReader for ParallelCommandReadBlockFromDB { self.chain.get_block_by_number(*num).ok().flatten() } }) - .collect::>() - ) + .collect::>()) + } + + fn query_txn_exec_state(&self, txn_hash: HashValue) -> String { + let txn = self + .chain + .get_transaction_info(txn_hash) + .expect("Query failed!"); + match txn { + Some(info) => match info.status { + KeptVMStatus::Executed => "OK".to_string(), + _ => "FALIED".to_string(), + }, + None => "CANT_FOUND_TXN".to_string(), + } } } @@ -321,7 +340,7 @@ impl ParallelCommandProgress { item_vec .into_iter() .map(|item| { - let (succeed, failed) = item.execute(command); + let (succeed, failed) = item.execute(self.block_reader.as_ref(), command); progress_bar.inc(1); CommandResult::new(succeed, failed.len()) }) @@ -355,7 +374,11 @@ impl ParallelCommandProgress { } pub trait ParallelCommand { - fn execute(&self, cmd: &CommandT) -> (usize, Vec); + fn execute( + &self, + block_reader: &dyn ParallelCommandBlockReader, + cmd: &CommandT, + ) -> (usize, Vec); fn matched(&self, filter: &Option) -> bool; } @@ -365,7 +388,8 @@ pub trait ParallelCommandObserver { fn after_progress(&self) -> Result<()>; } -pub trait ParallelCommandBlockReader { +pub trait ParallelCommandBlockReader: Sync + Send { fn get_progress_interval(&self) -> u64; fn read(&self, load_bar: &ProgressBar) -> Result>; + fn query_txn_exec_state(&self, txn_hash: HashValue) -> String; } diff --git a/cmd/db-exporter/src/commands/decode_payload.rs b/cmd/db-exporter/src/commands/decode_payload.rs index 54692d50df..cb0ea7c67e 100644 --- a/cmd/db-exporter/src/commands/decode_payload.rs +++ b/cmd/db-exporter/src/commands/decode_payload.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use crate::command_progress::{ - ParallelCommand, ParallelCommandFilter, ParallelCommandObserver, ParallelCommandProgress, - ParallelCommandReadBlockFromDB, + ParallelCommand, ParallelCommandBlockReader, ParallelCommandFilter, ParallelCommandObserver, + ParallelCommandProgress, ParallelCommandReadBlockFromDB, }; use anyhow::Result; use chrono::{DateTime, Utc}; @@ -104,6 +104,7 @@ pub struct CSVHeaders { func_name: String, ty_args: String, args: String, + execute_result: String, timestamp: u64, date_time: String, } @@ -135,7 +136,11 @@ fn timestamp_to_datetime(timestamp: u64) -> String { } impl ParallelCommand for Block { - fn execute(&self, command: &CommandDecodePayload) -> (usize, Vec) { + fn execute( + &self, + block_reader: &dyn ParallelCommandBlockReader, + command: &CommandDecodePayload, + ) -> (usize, Vec) { // let errors = vec![]; // let mut success_module_size = 0; @@ -152,6 +157,8 @@ impl ParallelCommand for Block starcoin_abi_decoder::decode_txn_payload(&statedb, txn.payload()) .expect("Decode transaction payload failed!"); + let execute_result = block_reader.query_txn_exec_state(txn.hash()); + let mut writer = command.writer_mutex.lock().unwrap(); match decoded_txn_payload { DecodedTransactionPayload::ScriptFunction(payload) => writer @@ -164,29 +171,41 @@ impl ParallelCommand for Block ty_args: payload .ty_args .iter() - .map(|a| a.to_string()) + .map(|a| a.to_string().trim_matches('"').to_string()) .collect::>() .join("|"), args: payload .args .iter() - .map(|a| a.0.to_string()) + .map(|a| a.0.to_string().trim_matches('"').to_string()) .collect::>() .join("|"), timestamp, + execute_result, date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), - DecodedTransactionPayload::Script(_) => writer + DecodedTransactionPayload::Script(script) => writer .serialize(CSVHeaders { block_num: block_num.clone(), txn_hash: txn.hash().to_string(), txn_type: String::from("Script"), signer, - func_name: "".to_string(), - ty_args: "".to_string(), - args: "".to_string(), + func_name: String::from(""), + ty_args: script + .ty_args + .iter() + .map(|a| a.to_string()) + .collect::>() + .join("|"), + args: script + .args + .iter() + .map(|a| a.0.to_string().trim_matches('"').to_string()) + .collect::>() + .join("|"), timestamp, + execute_result, date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), @@ -200,6 +219,7 @@ impl ParallelCommand for Block ty_args: "".to_string(), args: "".to_string(), timestamp, + execute_result, date_time: formatted_date.clone(), }) .expect("Write into CSV failed!"), @@ -219,21 +239,17 @@ impl ParallelCommand for Block }; match filters { - Some(filter) => { - self.transactions().iter().any(|txn| { - match txn.payload() { - TransactionPayload::ScriptFunction(payload) => { - filter.match_signer(&txn.sender().to_string()) - && filter.match_func_name(payload.function().as_str()) - && filter.match_ty_args(payload.ty_args()) - && filter.match_args(payload.args()) - }, - TransactionPayload::Script(_) | TransactionPayload::Package(_) => { - filter.match_signer(&txn.sender().to_string()) - }, - } - }) - }, + Some(filter) => self.transactions().iter().any(|txn| match txn.payload() { + TransactionPayload::ScriptFunction(payload) => { + filter.match_signer(&txn.sender().to_string()) + && filter.match_func_name(payload.function().as_str()) + && filter.match_ty_args(payload.ty_args()) + && filter.match_args(payload.args()) + } + TransactionPayload::Script(_) | TransactionPayload::Package(_) => { + filter.match_signer(&txn.sender().to_string()) + } + }), None => true, } } @@ -291,7 +307,7 @@ pub fn do_decode_payload( #[test] pub fn test_decode_payload() -> Result<()> { let input = PathBuf::from("/Users/bobong/.starcoin/main"); - let output = PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet/output.csv"); + let output = PathBuf::from("/Users/bobong/Downloads/unit-output.csv"); // do_decode_payload(Main, input.clone(), output.clone(), None, None, None)?; diff --git a/cmd/db-exporter/src/commands/mod.rs b/cmd/db-exporter/src/commands/mod.rs index e69de29bb2..411723f565 100644 --- a/cmd/db-exporter/src/commands/mod.rs +++ b/cmd/db-exporter/src/commands/mod.rs @@ -0,0 +1,3 @@ +pub mod decode_payload; +pub mod verify_header; +pub mod verify_module; diff --git a/cmd/db-exporter/src/commands/verify_header.rs b/cmd/db-exporter/src/commands/verify_header.rs index d8c8970f5b..c781eb7c61 100644 --- a/cmd/db-exporter/src/commands/verify_header.rs +++ b/cmd/db-exporter/src/commands/verify_header.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use std::sync::Arc; use crate::command_progress::{ - ParallelCommand, ParallelCommandFilter, ParallelCommandProgress, + ParallelCommand, ParallelCommandBlockReader, ParallelCommandFilter, ParallelCommandProgress, ParallelCommandReadBodyFromExportLine, }; use starcoin_types::block::Block; @@ -43,7 +43,11 @@ pub fn verify_header_via_export_file(path: PathBuf, batch_size: usize) -> anyhow } impl ParallelCommand for Block { - fn execute(&self, _cmd: &VerifyHeaderCmdType) -> (usize, Vec) { + fn execute( + &self, + _reader: &dyn ParallelCommandBlockReader, + _cmd: &VerifyHeaderCmdType, + ) -> (usize, Vec) { let ret = G_CRYPTONIGHT.verify_header_difficulty(self.header.difficulty(), &self.header); match ret { Ok(_) => (1, vec![]), diff --git a/cmd/db-exporter/src/commands/verify_module.rs b/cmd/db-exporter/src/commands/verify_module.rs index 20386bdde7..2e20c6369e 100644 --- a/cmd/db-exporter/src/commands/verify_module.rs +++ b/cmd/db-exporter/src/commands/verify_module.rs @@ -1,3 +1,7 @@ +use crate::command_progress::{ + ParallelCommand, ParallelCommandBlockReader, ParallelCommandFilter, ParallelCommandProgress, + ParallelCommandReadBodyFromExportLine, +}; use clap::Parser; use move_binary_format::errors::Location; use starcoin_crypto::HashValue; @@ -5,11 +9,6 @@ use starcoin_types::{block::Block, transaction::TransactionPayload}; use starcoin_vm_types::{errors::VMError, file_format::CompiledModule}; use std::sync::Arc; use std::{fmt::Debug, path::PathBuf}; -//use starcoin_accumulator::node::AccumulatorStoreType::Block; -use crate::command_progress::{ - ParallelCommand, ParallelCommandFilter, ParallelCommandProgress, - ParallelCommandReadBodyFromExportLine, -}; #[derive(Debug, Parser)] #[clap( @@ -32,7 +31,11 @@ pub struct VerifyModuleError { pub struct VerifyModulesType; impl ParallelCommand for Block { - fn execute(&self, _cmd: &VerifyModulesType) -> (usize, Vec) { + fn execute( + &self, + _reader: &dyn ParallelCommandBlockReader, + _cmd: &VerifyModulesType, + ) -> (usize, Vec) { let mut errors = vec![]; let mut success_modules = 0; let block = self; diff --git a/cmd/db-exporter/src/lib.rs b/cmd/db-exporter/src/lib.rs index 4eef6f976d..170251ca05 100644 --- a/cmd/db-exporter/src/lib.rs +++ b/cmd/db-exporter/src/lib.rs @@ -1,7 +1,5 @@ // Copyright (c) The Starcoin Core Contributors // SPDX-License-Identifier: Apache-2.0 -pub mod command_decode_payload; pub mod command_progress; -pub mod verify_header; -pub mod verify_module; +pub mod commands; diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs index f1330b3707..dd4f7aff22 100644 --- a/cmd/db-exporter/src/main.rs +++ b/cmd/db-exporter/src/main.rs @@ -5,11 +5,11 @@ use anyhow::{bail, format_err, Result}; use bcs_ext::{BCSCodec, Sample}; use clap::{IntoApp, Parser}; use csv::Writer; -use db_exporter::{ - command_decode_payload::{do_decode_payload_command, DecodePayloadCommandOptions}, - verify_header::{verify_header_via_export_file, VerifyHeaderOptions}, - verify_module::{verify_modules_via_export_file, VerifyModuleOptions}, -}; +// use db_exporter::{ +// decode_payload::{do_decode_payload_command, DecodePayloadCommandOptions}, +// verify_header::{verify_header_via_export_file, VerifyHeaderOptions}, +// verify_module::{verify_modules_via_export_file, VerifyModuleOptions}, +// }; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use serde::{ser::SerializeMap, Serialize, Serializer}; use starcoin_account_api::AccountInfo; @@ -70,6 +70,12 @@ use std::{ time::SystemTime, }; +use db_exporter::commands::{ + decode_payload::{do_decode_payload_command, DecodePayloadCommandOptions}, + verify_header::{verify_header_via_export_file, VerifyHeaderOptions}, + verify_module::{verify_modules_via_export_file, VerifyModuleOptions}, +}; + const BLOCK_GAP: u64 = 1000; const BACK_SIZE: u64 = 10000; const SNAP_GAP: u64 = 128; @@ -1887,6 +1893,7 @@ pub fn export_resource( let now2 = Instant::now(); for (k, v) in resource_set.iter() { let struct_tag = StructTag::decode(k.as_slice())?; + println!("struct_tag: {:?}", struct_tag.to_string()); if struct_tag == resource_struct_tag { let annotated_struct = value_annotator.view_struct(resource_struct_tag.clone(), v.as_slice())?; From 88e192ba836cdd2831a3ac45c46161db446749dd Mon Sep 17 00:00:00 2001 From: welbon Date: Thu, 22 Feb 2024 15:54:15 +0800 Subject: [PATCH 14/16] update move language modify --- Cargo.toml | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cb919071f6..74c80bea16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -333,31 +333,31 @@ log4rs = { version = "1.2.0", features = ["background_rotation", "gzip"] } lru = "0.7.8" merkletree = { version = "0.22.1" } mirai-annotations = "1.10.1" -move-binary-format = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-bytecode-source-map = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-bytecode-verifier = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-cli = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-command-line-common = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-compiler = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-core-types = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-coverage = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-disassembler = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-docgen = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-errmapgen = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-ir-compiler = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-ir-types = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-model = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-package = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-prover = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-prover-test-utils = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-resource-viewer = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-stdlib = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-transactional-test-runner = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-unit-test = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16", features = ["table-extension"] } -move-vm-runtime = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-vm-types = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-table-extension = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16" } -move-vm-test-utils = { git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16", features = ["table-extension"] } +move-binary-format = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-bytecode-source-map = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-bytecode-verifier = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-cli = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-command-line-common = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-compiler = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-core-types = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-coverage = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-disassembler = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-docgen = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-errmapgen = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-ir-compiler = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-ir-types = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-model = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-package = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-prover = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-prover-test-utils = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-resource-viewer = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-stdlib = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-transactional-test-runner = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-unit-test = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249", features = ["table-extension"] } +move-vm-runtime = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-vm-types = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-table-extension = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249" } +move-vm-test-utils = { git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249", features = ["table-extension"] } names = { version = "0.14.0", default-features = false } network-api = { path = "network/api", package = "network-api" } @@ -522,7 +522,7 @@ unsigned-varint = { version = "0.6.0", features = [ "futures", "asynchronous_codec", ] } -vm = { package = "move-binary-format", git = "https://github.com/starcoinorg/move", rev = "15d764f35c1131437b0e3be262bfa5f9b449ce16", features = [ +vm = { package = "move-binary-format", git = "https://github.com/starcoinorg/move", rev = "13bd7a8a279f1aecb4dc1e9c94e42da512320249", features = [ "fuzzing", ] } vm-status-translator = { path = "vm/vm-status-translator" } From b0b76c18ec0f7418ba3b6bb7a0547669ad3b493a Mon Sep 17 00:00:00 2001 From: welbon Date: Thu, 22 Feb 2024 16:56:33 +0800 Subject: [PATCH 15/16] add cargo.lock --- Cargo.lock | 78 +++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83951801a5..41f3d14257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -978,7 +978,7 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "bytecode-interpreter-crypto" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "curve25519-dalek-fiat", @@ -5018,7 +5018,7 @@ checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" [[package]] name = "move-abigen" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5035,7 +5035,7 @@ dependencies = [ [[package]] name = "move-binary-format" version = "0.0.3" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "arbitrary", @@ -5051,12 +5051,12 @@ dependencies = [ [[package]] name = "move-borrow-graph" version = "0.0.1" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" [[package]] name = "move-bytecode-source-map" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5071,7 +5071,7 @@ dependencies = [ [[package]] name = "move-bytecode-utils" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "move-binary-format", @@ -5083,7 +5083,7 @@ dependencies = [ [[package]] name = "move-bytecode-verifier" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "fail", @@ -5096,7 +5096,7 @@ dependencies = [ [[package]] name = "move-bytecode-viewer" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "clap 3.2.23", @@ -5113,7 +5113,7 @@ dependencies = [ [[package]] name = "move-cli" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5159,7 +5159,7 @@ dependencies = [ [[package]] name = "move-command-line-common" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "difference", @@ -5176,7 +5176,7 @@ dependencies = [ [[package]] name = "move-compiler" version = "0.0.1" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5205,7 +5205,7 @@ dependencies = [ [[package]] name = "move-core-types" version = "0.0.4" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "arbitrary", @@ -5229,7 +5229,7 @@ dependencies = [ [[package]] name = "move-coverage" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5266,7 +5266,7 @@ dependencies = [ [[package]] name = "move-disassembler" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "clap 3.2.23", @@ -5284,7 +5284,7 @@ dependencies = [ [[package]] name = "move-docgen" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "codespan 0.11.1", @@ -5302,7 +5302,7 @@ dependencies = [ [[package]] name = "move-errmapgen" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5316,7 +5316,7 @@ dependencies = [ [[package]] name = "move-ir-compiler" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5335,7 +5335,7 @@ dependencies = [ [[package]] name = "move-ir-to-bytecode" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "codespan-reporting", @@ -5354,7 +5354,7 @@ dependencies = [ [[package]] name = "move-ir-to-bytecode-syntax" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "hex", @@ -5367,7 +5367,7 @@ dependencies = [ [[package]] name = "move-ir-types" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "hex", @@ -5381,7 +5381,7 @@ dependencies = [ [[package]] name = "move-model" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "codespan 0.11.1", @@ -5407,7 +5407,7 @@ dependencies = [ [[package]] name = "move-package" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5491,7 +5491,7 @@ dependencies = [ [[package]] name = "move-prover" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "async-trait", @@ -5528,7 +5528,7 @@ dependencies = [ [[package]] name = "move-prover-boogie-backend" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "async-trait", @@ -5556,7 +5556,7 @@ dependencies = [ [[package]] name = "move-prover-test-utils" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "move-command-line-common", @@ -5567,7 +5567,7 @@ dependencies = [ [[package]] name = "move-read-write-set-types" version = "0.0.3" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "move-binary-format", @@ -5578,7 +5578,7 @@ dependencies = [ [[package]] name = "move-resource-viewer" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5593,7 +5593,7 @@ dependencies = [ [[package]] name = "move-stackless-bytecode" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "codespan 0.11.1", "codespan-reporting", @@ -5620,7 +5620,7 @@ dependencies = [ [[package]] name = "move-stackless-bytecode-interpreter" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bytecode-interpreter-crypto", @@ -5638,7 +5638,7 @@ dependencies = [ [[package]] name = "move-stdlib" version = "0.1.1" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "hex", @@ -5661,7 +5661,7 @@ dependencies = [ [[package]] name = "move-symbol-pool" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "once_cell", "serde 1.0.152", @@ -5670,7 +5670,7 @@ dependencies = [ [[package]] name = "move-table-extension" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "bcs", @@ -5687,7 +5687,7 @@ dependencies = [ [[package]] name = "move-transactional-test-runner" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "clap 3.2.23", @@ -5722,7 +5722,7 @@ dependencies = [ [[package]] name = "move-unit-test" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "better_any", @@ -5753,7 +5753,7 @@ dependencies = [ [[package]] name = "move-vm-runtime" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "better_any", "fail", @@ -5770,7 +5770,7 @@ dependencies = [ [[package]] name = "move-vm-test-utils" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "move-binary-format", @@ -5784,7 +5784,7 @@ dependencies = [ [[package]] name = "move-vm-types" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "bcs", "move-binary-format", @@ -7814,7 +7814,7 @@ dependencies = [ [[package]] name = "read-write-set" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "move-binary-format", @@ -7829,7 +7829,7 @@ dependencies = [ [[package]] name = "read-write-set-dynamic" version = "0.1.0" -source = "git+https://github.com/starcoinorg/move?rev=15d764f35c1131437b0e3be262bfa5f9b449ce16#15d764f35c1131437b0e3be262bfa5f9b449ce16" +source = "git+https://github.com/starcoinorg/move?rev=13bd7a8a279f1aecb4dc1e9c94e42da512320249#13bd7a8a279f1aecb4dc1e9c94e42da512320249" dependencies = [ "anyhow", "move-binary-format", From ceec189e961e93e5448aa187d9819112724c06dd Mon Sep 17 00:00:00 2001 From: welbon Date: Thu, 22 Feb 2024 18:26:56 +0800 Subject: [PATCH 16/16] fixed the query block number --- cmd/db-exporter/src/command_progress.rs | 68 +++++++++++++++---------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 558d462cae..f5487ea513 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, format_err, Result}; +use anyhow::{bail, Result}; use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::*; use starcoin_chain::{BlockChain, ChainReader}; @@ -135,6 +135,45 @@ impl ParallelCommandBlockReader for ParallelCommandReadBodyFromExportLine { } } +fn start_end_check(start: u64, end: u64, cur_num: u64) -> (u64, u64) { + if start == 0 && end == 0 { + (0, cur_num) + } else { + assert!(start < end, "End must bigger than Start"); + assert!(end <= cur_num, "End number should less than Head num"); + (start, end) + } +} + +#[test] +fn test_start_end_check() { + let (start, end) = start_end_check(0, 100, 100); + assert_eq!(start, 0); + assert_eq!(end, 100); + + let (start, end) = start_end_check(0, 0, 100); + assert_eq!(start, 0); + assert_eq!(end, 100); +} + +#[test] +#[should_panic = "End must bigger than Start"] +fn test_start_end_check_panic_invalid() { + start_end_check(100, 1, 100); +} + +#[test] +#[should_panic = "End must bigger than Start"] +fn test_start_end_check_panic_invalid2() { + start_end_check(100, 100, 100); +} + +#[test] +#[should_panic = "End number should less than Head num"] +fn test_start_end_check_panic_overflow() { + start_end_check(100, 500, 100); +} + pub struct ParallelCommandReadBlockFromDB { start_num: u64, end_num: u64, @@ -142,8 +181,6 @@ pub struct ParallelCommandReadBlockFromDB { skip_empty_block: bool, } -const BLOCK_GAP: u64 = 1000; - impl ParallelCommandReadBlockFromDB { pub fn new( input_path: PathBuf, @@ -164,30 +201,7 @@ impl ParallelCommandReadBlockFromDB { ) .expect("Failed to initialize block chain"); - let cur_num = chain.status().head().number(); - - let (start_num, end_num) = if start == 0 && end == 0 { - (0, cur_num) - } else { - let final_end = if cur_num > end + BLOCK_GAP { - end - } else if cur_num > BLOCK_GAP { - cur_num - BLOCK_GAP - } else { - end - }; - (start, final_end) - }; - - if start > cur_num || start > end { - return Err(format_err!( - "cur_num {} start {} end {} illegal", - cur_num, - start, - end - )); - }; - + let (start_num, end_num) = start_end_check(start, end, chain.status().head().number()); Ok(( Self { start_num,