From 25cb69dace8486250b24528549905abed4cfcb64 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 12 Apr 2024 18:38:23 +0000 Subject: [PATCH 01/54] add _generate_tbl_fast target --- examples/CMakeLists.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index bc381ce83..f48e326eb 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -34,6 +34,7 @@ function(assign_ir) COMMAND $ -b ${binary_name} -c circuit_${target}.crct + -j table_pieces_${target}.json -e ${curve_type} --generate-type circuit ${max_num_provers_flag} ${max_num_provers_amount} @@ -48,6 +49,7 @@ function(assign_ir) -i ${INPUTS_DIR}/${input} ${minus_p} ${private_input_string} -c circuit_${target}.crct + -j table_pieces_${target}.json -t assignment_${target}.tbl -e ${curve_type} --check --generate-type circuit-assignment ${max_num_provers_flag} ${max_num_provers_amount} @@ -62,6 +64,7 @@ function(assign_ir) -i ${INPUTS_DIR}/${input} ${minus_p} ${private_input_string} -c circuit_${target}.crct + -j table_pieces_${target}.json -t assignment_${target}.tbl -e ${curve_type} --generate-type circuit-assignment ${max_num_provers_flag} ${max_num_provers_amount} @@ -83,6 +86,20 @@ function(assign_ir) COMMAND_EXPAND_LISTS VERBATIM) + add_custom_target(${target}_generate_tbl_fast + COMMAND $ + -b ${binary_name} + -i ${INPUTS_DIR}/${input} + ${minus_p} ${private_input_string} + -j table_pieces_${target}.json + -t assignment_${target}.tbl -e ${curve_type} --check + --generate-type assignment-fast + ${max_num_provers_flag} ${max_num_provers_amount} + ${arithmetization_flag} ${arithmetization_amount} + DEPENDS ${target} ${INPUTS_DIR}/${input} $ + COMMAND_EXPAND_LISTS + VERBATIM) + add_custom_target(${target}_generate_tbl_no_check COMMAND $ -b ${binary_name} From 14d8a63d5e96c214650b703e48168700cb36579b Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 12 Apr 2024 18:39:04 +0000 Subject: [PATCH 02/54] modify examples(reset befor merge into master) --- examples/cpp/merkle_tree_poseidon.cpp | 90 +- examples/cpp/poseidon.cpp | 4 +- examples/inputs/merkle_tree_poseidon.inp | 2019 +++++++++++++++++++++- 3 files changed, 2087 insertions(+), 26 deletions(-) diff --git a/examples/cpp/merkle_tree_poseidon.cpp b/examples/cpp/merkle_tree_poseidon.cpp index 4ca9e29a2..814bd8bd9 100644 --- a/examples/cpp/merkle_tree_poseidon.cpp +++ b/examples/cpp/merkle_tree_poseidon.cpp @@ -6,39 +6,81 @@ using namespace nil::crypto3; using namespace nil::crypto3::algebra::curves; [[circuit]] pallas::base_field_type::value_type merkle_tree_poseidon ( - std::array layer_0_leaves) { - - std::array layer_1_leaves; - std::size_t layer_1_size = 0x10; - std::array layer_2_leaves; - std::size_t layer_2_size = 0x8; - std::array layer_3_leaves; - std::size_t layer_3_size = 0x4; - std::array layer_4_leaves; - std::size_t layer_4_size = 0x2; + std::array layer_100_leaves) { + + // std::array layer_400_leaves; + // std::size_t layer_400_size = 0x400; + // std::array layer_200_leaves; + // std::size_t layer_200_size = 0x200; + // std::array layer_100_leaves; + // std::size_t layer_100_size = 0x100; + std::array layer_080_leaves; + std::size_t layer_080_size = 0x080; + std::array layer_040_leaves; + std::size_t layer_040_size = 0x040; + std::array layer_020_leaves; + std::size_t layer_020_size = 0x020; + std::array layer_010_leaves; + std::size_t layer_010_size = 0x010; + std::array layer_008_leaves; + std::size_t layer_008_size = 0x008; + std::array layer_004_leaves; + std::size_t layer_004_size = 0x004; + std::array layer_002_leaves; + std::size_t layer_002_size = 0x002; typename pallas::base_field_type::value_type root; - for (std::size_t leaf_index = 0; leaf_index < layer_1_size; leaf_index++) { - layer_1_leaves[leaf_index] = - hash(layer_0_leaves[2 * leaf_index], layer_0_leaves[2 * leaf_index + 1]); + + // for (std::size_t leaf_index = 0; leaf_index < layer_400_size; leaf_index++) { + // layer_400_leaves[leaf_index] = + // hash(input[2 * leaf_index], input[2 * leaf_index + 1]); + // } + + // for (std::size_t leaf_index = 0; leaf_index < layer_200_size; leaf_index++) { + // layer_200_leaves[leaf_index] = + // hash(layer_400_leaves[2 * leaf_index], layer_400_leaves[2 * leaf_index + 1]); + // } + + // for (std::size_t leaf_index = 0; leaf_index < layer_100_size; leaf_index++) { + // layer_100_leaves[leaf_index] = + // hash(layer_200_leaves[2 * leaf_index], layer_200_leaves[2 * leaf_index + 1]); + // } + + for (std::size_t leaf_index = 0; leaf_index < layer_080_size; leaf_index++) { + layer_080_leaves[leaf_index] = + hash(layer_100_leaves[2 * leaf_index], layer_100_leaves[2 * leaf_index + 1]); } - for (std::size_t leaf_index = 0; leaf_index < layer_2_size; leaf_index++) { - layer_2_leaves[leaf_index] = - hash(layer_1_leaves[2 * leaf_index], layer_1_leaves[2 * leaf_index + 1]); + for (std::size_t leaf_index = 0; leaf_index < layer_040_size; leaf_index++) { + layer_040_leaves[leaf_index] = + hash(layer_080_leaves[2 * leaf_index], layer_080_leaves[2 * leaf_index + 1]); } - for (std::size_t leaf_index = 0; leaf_index < layer_3_size; leaf_index++) { - layer_3_leaves[leaf_index] = - hash(layer_2_leaves[2 * leaf_index], layer_2_leaves[2 * leaf_index + 1]); + for (std::size_t leaf_index = 0; leaf_index < layer_020_size; leaf_index++) { + layer_020_leaves[leaf_index] = + hash(layer_040_leaves[2 * leaf_index], layer_040_leaves[2 * leaf_index + 1]); } - for (std::size_t leaf_index = 0; leaf_index < layer_4_size; leaf_index++) { - layer_4_leaves[leaf_index] = - hash(layer_3_leaves[2 * leaf_index], layer_3_leaves[2 * leaf_index + 1]); + for (std::size_t leaf_index = 0; leaf_index < layer_010_size; leaf_index++) { + layer_010_leaves[leaf_index] = + hash(layer_020_leaves[2 * leaf_index], layer_020_leaves[2 * leaf_index + 1]); } - root = hash(layer_4_leaves[0], layer_4_leaves[1]); + for (std::size_t leaf_index = 0; leaf_index < layer_008_size; leaf_index++) { + layer_008_leaves[leaf_index] = + hash(layer_010_leaves[2 * leaf_index], layer_010_leaves[2 * leaf_index + 1]); + } + + for (std::size_t leaf_index = 0; leaf_index < layer_004_size; leaf_index++) { + layer_004_leaves[leaf_index] = + hash(layer_008_leaves[2 * leaf_index], layer_008_leaves[2 * leaf_index + 1]); + } + + for (std::size_t leaf_index = 0; leaf_index < layer_002_size; leaf_index++) { + layer_002_leaves[leaf_index] = + hash(layer_004_leaves[2 * leaf_index], layer_004_leaves[2 * leaf_index + 1]); + } - return root; + typename pallas::base_field_type::value_type real_root = hash(layer_002_leaves[0], layer_002_leaves[1]); + return real_root; } diff --git a/examples/cpp/poseidon.cpp b/examples/cpp/poseidon.cpp index e997bcda8..f0ff531d8 100644 --- a/examples/cpp/poseidon.cpp +++ b/examples/cpp/poseidon.cpp @@ -8,6 +8,8 @@ using namespace nil::crypto3::algebra::curves; typename pallas::base_field_type::value_type c) { typename pallas::base_field_type::value_type hash_result = hash(b, c); + typename pallas::base_field_type::value_type hash_result_2 = hash(hash_result, c); + typename pallas::base_field_type::value_type hash_result_3 = hash(hash_result_2, hash_result); - return hash_result; + return hash_result_3; } diff --git a/examples/inputs/merkle_tree_poseidon.inp b/examples/inputs/merkle_tree_poseidon.inp index a9ab5f73c..1803c3c6a 100644 --- a/examples/inputs/merkle_tree_poseidon.inp +++ b/examples/inputs/merkle_tree_poseidon.inp @@ -30,4 +30,2021 @@ {"field":29}, {"field":30}, {"field":31}, - {"field":32}] } ] + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32}, + {"field":1}, + {"field":2}, + {"field":3}, + {"field":4}, + {"field":5}, + {"field":6}, + {"field":7}, + {"field":8}, + {"field":9}, + {"field":10}, + {"field":11}, + {"field":12}, + {"field":13}, + {"field":14}, + {"field":15}, + {"field":16}, + {"field":17}, + {"field":18}, + {"field":19}, + {"field":20}, + {"field":21}, + {"field":22}, + {"field":23}, + {"field":24}, + {"field":25}, + {"field":26}, + {"field":27}, + {"field":28}, + {"field":29}, + {"field":30}, + {"field":31}, + {"field":32} + ] } ] From 447bca11aab66e6e6b7448bca21dd46fb224433b Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 12 Apr 2024 18:44:31 +0000 Subject: [PATCH 03/54] add table_pieces handling into assigner main --- bin/assigner/src/main.cpp | 96 +++++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 07e423603..ce65819d0 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -487,6 +487,7 @@ int curve_dependent_main(std::string bytecode_file_name, std::string private_input_file_name, std::string assignment_table_file_name, std::string circuit_file_name, + std::string table_pieces_file_name, std::string processed_public_input_file_name, long stack_size, bool check_validity, @@ -534,6 +535,27 @@ int curve_dependent_main(std::string bytecode_file_name, return 1; } + using var = crypto3::zk::snark::plonk_variable; + std::vector> table_pieces = {}; // we create table pieces in any case and pass into assigner by reference + if (gen_mode.has_fast_tbl()) { // if we are generating tables in a fast way then need to parse table_pieces from file + + std::ifstream inp_json(table_pieces_file_name); + + if (!inp_json.is_open()) { + std::cerr << "unable to open table_pieces file" << std::endl; + return 1; + } + + std::string str((std::istreambuf_iterator(inp_json)), + std::istreambuf_iterator()); + auto parsed = boost::json::parse(str); + boost::json::array arr = parsed.as_array(); + + for (const auto& item : arr) { + table_pieces.emplace_back(item.as_object()); + } + } + auto assigner_instance_creation_start = std::chrono::high_resolution_clock::now(); nil::blueprint::assigner assigner_instance( desc, @@ -563,7 +585,13 @@ int curve_dependent_main(std::string bytecode_file_name, BOOST_LOG_TRIVIAL(debug) << "parse_ir_file_duration: " << parse_ir_file_duration.count() << "ms"; auto parser_evaluation_start = std::chrono::high_resolution_clock::now(); - if (!assigner_instance.evaluate(public_input_json_value.as_array(), private_input_json_value.as_array())) { + if ( + !assigner_instance.evaluate( + public_input_json_value.as_array(), + private_input_json_value.as_array(), + table_pieces + ) + ) { return 1; } auto parser_evaluation_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - parser_evaluation_start); @@ -575,6 +603,20 @@ int curve_dependent_main(std::string bytecode_file_name, return 0; } + if (gen_mode.has_circuit()) { // if we are generation circuit then we are generation table pieces in the same time. Need to write itno file + boost::json::array pieces_json; + // for (const auto& piece : nil::blueprint::table_pieces) { + for (const auto& piece : table_pieces) { + pieces_json.push_back(piece.to_json()); + } + + std::string serialized = boost::json::serialize(pieces_json); + + std::ofstream file(table_pieces_file_name); + file << serialized; + file.close(); + } + auto pack_lookup_start = std::chrono::high_resolution_clock::now(); // pack lookup tables if (assigner_instance.circuits[0].get_reserved_tables().size() > 0) { @@ -744,6 +786,7 @@ int main(int argc, char *argv[]) { ("private-input,p", boost::program_options::value(), "Private input file") ("assignment-table,t", boost::program_options::value(), "Assignment table output file") ("circuit,c", boost::program_options::value(), "Circuit output file") + ("table-pieces,j", boost::program_options::value(), "Table pieces json file") ("input-column", boost::program_options::value(), "Output file for public input column") ("elliptic-curve-type,e", boost::program_options::value(), "Native elliptic curve type (pallas, vesta, ed25519, bls12381)") ("stack-size,s", boost::program_options::value(), "Stack size in bytes") @@ -752,7 +795,7 @@ int main(int argc, char *argv[]) { ("print_circuit_output", "deprecated, use \"-f\" instead") ("print-circuit-output-format,f", boost::program_options::value(), "print output of the circuit (dec, hex)") ("policy", boost::program_options::value(), "Policy for creating circuits. Possible values: default") - ("generate-type", boost::program_options::value(), "Define generated output. Possible values: circuit, assignment, circuit-assignment, public-input-column, size_estimation(does not generate anything, just evaluates circuit size). Default value is circuit-assignment") + ("generate-type", boost::program_options::value(), "Define generated output. Possible values: circuit, assignment, assignment-fast, circuit-assignment, public-input-column, size_estimation(does not generate anything, just evaluates circuit size). Default value is circuit-assignment") ("max-num-provers", boost::program_options::value(), "Maximum number of provers. Possible values >= 1") ("max-lookup-rows", boost::program_options::value(), "Maximum number of lookup rows") ("target-prover", boost::program_options::value(), "Assignment table and circuit will be generated only for defined prover. Possible values [0, max-num-provers)") @@ -794,6 +837,7 @@ int main(int argc, char *argv[]) { std::string private_input_file_name; std::string assignment_table_file_name; std::string circuit_file_name; + std::string table_pieces_file_name; std::string processed_public_input_file_name; std::string elliptic_curve; nil::blueprint::print_format circuit_output_print_format; @@ -815,6 +859,8 @@ int main(int argc, char *argv[]) { gen_mode = nil::blueprint::generation_mode::circuit(); } else if (generate_type == "assignment") { gen_mode = nil::blueprint::generation_mode::assignments(); + } else if (generate_type == "assignment-fast") { + gen_mode = nil::blueprint::generation_mode::fast_tbl(); } else if (generate_type == "size_estimation") { gen_mode = nil::blueprint::generation_mode::size_estimation(); } else if (generate_type == "public-input-column") { @@ -869,6 +915,14 @@ int main(int argc, char *argv[]) { } } + if (vm.count("table-pieces")) { + table_pieces_file_name = vm["table-pieces"].as(); + } else { + std::cerr << "Invalid command line argument - table-pieces file name is not specified" << std::endl; + std::cout << options_desc << std::endl; + return 1; + } + if (vm.count("elliptic-curve-type")) { elliptic_curve = vm["elliptic-curve-type"].as(); } else { @@ -1010,6 +1064,7 @@ int main(int argc, char *argv[]) { private_input_file_name, assignment_table_file_name, circuit_file_name, + table_pieces_file_name, processed_public_input_file_name, stack_size, vm.count("check"), @@ -1033,24 +1088,25 @@ int main(int argc, char *argv[]) { break; } case 3: { - return curve_dependent_main>( - bytecode_file_name, - public_input_file_name, - private_input_file_name, - assignment_table_file_name, - circuit_file_name, - processed_public_input_file_name, - stack_size, - vm.count("check"), - log_options[log_level], - policy, - gen_mode, - max_num_provers, - max_lookup_rows, - target_prover, - circuit_output_print_format, - column_sizes - ); + UNREACHABLE("bls12381 curve based circuits are not supported yet"); + // return curve_dependent_main>( + // bytecode_file_name, + // public_input_file_name, + // private_input_file_name, + // assignment_table_file_name, + // circuit_file_name, + // processed_public_input_file_name, + // stack_size, + // vm.count("check"), + // log_options[log_level], + // policy, + // gen_mode, + // max_num_provers, + // max_lookup_rows, + // target_prover, + // circuit_output_print_format, + // column_sizes + // ); break; } }; From 359105ed8b913e16aaf9c423af31ba1e9a9aab95 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 16 Apr 2024 12:49:05 +0000 Subject: [PATCH 04/54] update blueprint --- libs/blueprint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/blueprint b/libs/blueprint index 5a2209e09..b05f6c370 160000 --- a/libs/blueprint +++ b/libs/blueprint @@ -1 +1 @@ -Subproject commit 5a2209e09c1e19ad41b1d3ff308185ea03a3c114 +Subproject commit b05f6c370b9301b14b4ce6109fefe6f1ddb96da1 From 0498047b2b24017637a290a9fb7c2d1042be5005 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 16 Apr 2024 12:51:51 +0000 Subject: [PATCH 05/54] temporary small main.cpp change --- bin/assigner/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index ce65819d0..d73c8e688 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -606,7 +606,7 @@ int curve_dependent_main(std::string bytecode_file_name, if (gen_mode.has_circuit()) { // if we are generation circuit then we are generation table pieces in the same time. Need to write itno file boost::json::array pieces_json; // for (const auto& piece : nil::blueprint::table_pieces) { - for (const auto& piece : table_pieces) { + for (const auto& piece : nil::blueprint::table_pieces) { pieces_json.push_back(piece.to_json()); } From 0d62826420935f0a84a1332c4f6ff80c008c401a Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 18 Apr 2024 16:09:37 +0000 Subject: [PATCH 06/54] temporarily add assignments gen mode for the fast tbl generation --- bin/assigner/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index d73c8e688..18cf3cd74 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -860,7 +860,7 @@ int main(int argc, char *argv[]) { } else if (generate_type == "assignment") { gen_mode = nil::blueprint::generation_mode::assignments(); } else if (generate_type == "assignment-fast") { - gen_mode = nil::blueprint::generation_mode::fast_tbl(); + gen_mode = nil::blueprint::generation_mode::fast_tbl() | nil::blueprint::generation_mode::assignments(); } else if (generate_type == "size_estimation") { gen_mode = nil::blueprint::generation_mode::size_estimation(); } else if (generate_type == "public-input-column") { From aff72f0ffa605f94eb5e1e7a8a51816ab69dc493 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 16 May 2024 14:07:30 +0000 Subject: [PATCH 07/54] store different column types in different files, read constants from file in fast gen mode --- bin/assigner/src/main.cpp | 448 ++++++++++++++++++++++++++------------ 1 file changed, 310 insertions(+), 138 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 18cf3cd74..7185ec7dd 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -228,113 +228,37 @@ void print_vector_value( } } - template -void print_assignment_table(const assignment_proxy &table_proxy, +void print_selectors(const assignment_proxy &table_proxy, print_table_kind print_kind, - std::uint32_t ComponentConstantColumns, std::uint32_t ComponentSelectorColumns, + std::size_t padded_rows_amount, + std::uint32_t ComponentSelectorColumns, std::ostream &out = std::cout) { - using AssignmentTableType = assignment_proxy; - std::uint32_t usable_rows_amount; - std::uint32_t total_columns; - std::uint32_t total_size; - std::uint32_t shared_size = (print_kind == print_table_kind::MULTI_PROVER) ? 1 : 0; - std::uint32_t public_input_size = table_proxy.public_inputs_amount(); - std::uint32_t witness_size = table_proxy.witnesses_amount(); - std::uint32_t constant_size = table_proxy.constants_amount(); std::uint32_t selector_size = table_proxy.selectors_amount(); - const auto lookup_constant_cols = table_proxy.get_lookup_constant_cols(); - const auto lookup_selector_cols = table_proxy.get_lookup_selector_cols(); - - std::uint32_t max_public_inputs_size = 0; - std::uint32_t max_constant_size = 0; - std::uint32_t max_selector_size = 0; - - for (std::uint32_t i = 0; i < public_input_size; i++) { - max_public_inputs_size = std::max(max_public_inputs_size, table_proxy.public_input_column_size(i)); - } - - if (print_kind == print_table_kind::MULTI_PROVER) { - total_columns = witness_size + shared_size + public_input_size + constant_size + selector_size; - std::uint32_t max_shared_size = 0; - for (std::uint32_t i = 0; i < shared_size; i++) { - max_shared_size = std::max(max_shared_size, table_proxy.shared_column_size(i)); - } - for (const auto &i : lookup_constant_cols) { - max_constant_size = std::max(max_constant_size, table_proxy.constant_column_size(i)); - } - for (const auto &i : lookup_selector_cols) { - max_selector_size = std::max(max_selector_size, table_proxy.selector_column_size(i)); - } - usable_rows_amount = table_proxy.get_used_rows().size(); - usable_rows_amount = std::max({usable_rows_amount, max_shared_size, max_public_inputs_size, max_constant_size, max_selector_size}); - } else { // SINGLE_PROVER - total_columns = witness_size + shared_size + public_input_size + constant_size + selector_size; - std::uint32_t max_witness_size = 0; - for (std::uint32_t i = 0; i < witness_size; i++) { - max_witness_size = std::max(max_witness_size, table_proxy.witness_column_size(i)); - } - for (std::uint32_t i = 0; i < constant_size; i++) { - max_constant_size = std::max(max_constant_size, table_proxy.constant_column_size(i)); - } - for (std::uint32_t i = 0; i < selector_size; i++) { - max_selector_size = std::max(max_selector_size, table_proxy.selector_column_size(i)); - } - usable_rows_amount = std::max({max_witness_size, max_public_inputs_size, max_constant_size, max_selector_size}); - } - - std::uint32_t padded_rows_amount = std::pow(2, std::ceil(std::log2(usable_rows_amount))); - if (padded_rows_amount == usable_rows_amount) { - padded_rows_amount *= 2; - } - if (padded_rows_amount < 8) { - padded_rows_amount = 8; - } - total_size = padded_rows_amount * total_columns; - - using TTypeBase = nil::marshalling::field_type; - using table_value_marshalling_type = - nil::crypto3::marshalling::types::plonk_assignment_table; using column_type = typename crypto3::zk::snark::plonk_column; - print_size_t(witness_size, out); - print_size_t(public_input_size + shared_size, out); - print_size_t(constant_size, out); - print_size_t(selector_size, out); - print_size_t(usable_rows_amount, out); - print_size_t(padded_rows_amount, out); - - if (print_kind == print_table_kind::SINGLE_PROVER) { - print_size_t(witness_size * padded_rows_amount, out); - for (std::uint32_t i = 0; i < witness_size; i++) { - print_vector_value(padded_rows_amount, table_proxy.witness(i), out); - } - print_size_t((public_input_size + shared_size) * padded_rows_amount, out); - for (std::uint32_t i = 0; i < public_input_size; i++) { - print_vector_value(padded_rows_amount, table_proxy.public_input(i), out); - } - print_size_t(constant_size * padded_rows_amount, out); - for (std::uint32_t i = 0; i < constant_size; i++) { - print_vector_value(padded_rows_amount, table_proxy.constant(i), out); - } print_size_t(selector_size * padded_rows_amount, out); + if (print_kind == print_table_kind::SINGLE_PROVER) { for (std::uint32_t i = 0; i < selector_size; i++) { print_vector_value(padded_rows_amount, table_proxy.selector(i), out); } } else { const auto& rows = table_proxy.get_used_rows(); const auto& selector_rows = table_proxy.get_used_selector_rows(); - std::uint32_t witness_idx = 0; - // witness - print_size_t(witness_size * padded_rows_amount, out); - for( std::size_t i = 0; i < witness_size; i++ ){ - const auto column_size = table_proxy.witness_column_size(i); + print_size_t(selector_size * padded_rows_amount, out); + std::uint32_t selector_idx = 0; + for (std::uint32_t i = 0; i < ComponentSelectorColumns; i++) { + const auto column_size = table_proxy.selector_column_size(i); std::uint32_t offset = 0; for(const auto& j : rows){ if (j < column_size) { - print_field(table_proxy.witness(i, j), out); + if (selector_rows.find(j) != selector_rows.end()) { + print_field(table_proxy.selector(i, j), out); + } else { + print_zero_field(out); + } offset++; } } @@ -343,21 +267,34 @@ void print_assignment_table(const assignment_proxy &table_p print_zero_field(out); offset++; } - witness_idx += padded_rows_amount; + + selector_idx += padded_rows_amount; } - // public input - std::uint32_t pub_inp_idx = 0; - print_size_t((public_input_size + shared_size) * padded_rows_amount, out); - for (std::uint32_t i = 0; i < public_input_size; i++) { - print_vector_value(padded_rows_amount, table_proxy.public_input(i), out); - pub_inp_idx += padded_rows_amount; + + for (std::uint32_t i = ComponentSelectorColumns; i < selector_size; i++) { + print_vector_value(padded_rows_amount, table_proxy.selector(i), out); + selector_idx += padded_rows_amount; } - for (std::uint32_t i = 0; i < shared_size; i++) { - print_vector_value(padded_rows_amount, table_proxy.shared(i), out); - pub_inp_idx += padded_rows_amount; + } +} + +template +void print_constants(const assignment_proxy &table_proxy, + print_table_kind print_kind, + std::size_t padded_rows_amount, + std::uint32_t ComponentConstantColumns, + std::ostream &out = std::cout) { + std::uint32_t constant_size = table_proxy.constants_amount(); + + using column_type = typename crypto3::zk::snark::plonk_column; + + print_size_t(constant_size * padded_rows_amount, out); + if (print_kind == print_table_kind::SINGLE_PROVER) { + for (std::uint32_t i = 0; i < constant_size; i++) { + print_vector_value(padded_rows_amount, table_proxy.constant(i), out); } - // constant - print_size_t(constant_size * padded_rows_amount, out); + } else { + const auto& rows = table_proxy.get_used_rows(); std::uint32_t constant_idx = 0; for (std::uint32_t i = 0; i < ComponentConstantColumns; i++) { const auto column_size = table_proxy.constant_column_size(i); @@ -381,20 +318,61 @@ void print_assignment_table(const assignment_proxy &table_p print_vector_value(padded_rows_amount, table_proxy.constant(i), out); constant_idx += padded_rows_amount; } + } +} - // selector - print_size_t(selector_size * padded_rows_amount, out); - std::uint32_t selector_idx = 0; - for (std::uint32_t i = 0; i < ComponentSelectorColumns; i++) { - const auto column_size = table_proxy.selector_column_size(i); +template +void print_pub_inp(const assignment_proxy &table_proxy, + print_table_kind print_kind, + std::size_t padded_rows_amount, + std::ostream &out = std::cout) { + std::uint32_t shared_size = (print_kind == print_table_kind::MULTI_PROVER) ? 1 : 0; + std::uint32_t public_input_size = table_proxy.public_inputs_amount(); + + using column_type = typename crypto3::zk::snark::plonk_column; + + print_size_t((public_input_size + shared_size) * padded_rows_amount, out); + if (print_kind == print_table_kind::SINGLE_PROVER) { + for (std::uint32_t i = 0; i < public_input_size; i++) { + print_vector_value(padded_rows_amount, table_proxy.public_input(i), out); + } + } else { + + std::uint32_t pub_inp_idx = 0; + for (std::uint32_t i = 0; i < public_input_size; i++) { + print_vector_value(padded_rows_amount, table_proxy.public_input(i), out); + pub_inp_idx += padded_rows_amount; + } + for (std::uint32_t i = 0; i < shared_size; i++) { + print_vector_value(padded_rows_amount, table_proxy.shared(i), out); + pub_inp_idx += padded_rows_amount; + } + } +} + +template +void print_witness(const assignment_proxy &table_proxy, + print_table_kind print_kind, + std::size_t padded_rows_amount, + std::ostream &out = std::cout) { + std::uint32_t witness_size = table_proxy.witnesses_amount(); + + using column_type = typename crypto3::zk::snark::plonk_column; + + print_size_t(witness_size * padded_rows_amount, out); + if (print_kind == print_table_kind::SINGLE_PROVER) { + for (std::uint32_t i = 0; i < witness_size; i++) { + print_vector_value(padded_rows_amount, table_proxy.witness(i), out); + } + } else { + const auto& rows = table_proxy.get_used_rows(); + std::uint32_t witness_idx = 0; + for( std::size_t i = 0; i < witness_size; i++ ){ + const auto column_size = table_proxy.witness_column_size(i); std::uint32_t offset = 0; for(const auto& j : rows){ if (j < column_size) { - if (selector_rows.find(j) != selector_rows.end()) { - print_field(table_proxy.selector(i, j), out); - } else { - print_zero_field(out); - } + print_field(table_proxy.witness(i, j), out); offset++; } } @@ -403,16 +381,159 @@ void print_assignment_table(const assignment_proxy &table_p print_zero_field(out); offset++; } - - selector_idx += padded_rows_amount; + witness_idx += padded_rows_amount; } + } +} - for (std::uint32_t i = ComponentSelectorColumns; i < selector_size; i++) { - print_vector_value(padded_rows_amount, table_proxy.selector(i), out); - selector_idx += padded_rows_amount; +template +void print_assignment_table_fast( + const assignment_proxy &table_proxy, + print_table_kind print_kind, + std::string assignment_table_file_name +) { + + std::ifstream itable_header; + std::ofstream otable_witness; + std::ofstream otable_pub_inp; + itable_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!itable_header) { + throw std::runtime_error("Something wrong with input file header_" + assignment_table_file_name); // TODO: add catch + } + using TTypeBase = nil::marshalling::field_type; + using table_header_marshalling_type = nil::crypto3::marshalling::types::table_header_type; + + table_header_marshalling_type marshalled_table_header; + std::vector header_binary; + itable_header.seekg(0, std::ios_base::end); + const auto header_size = itable_header.tellg(); + header_binary.resize(header_size); + auto hiter = header_binary.begin(); + auto status = marshalled_table_header.read(hiter, header_binary.size()); +// TODO add status check + + std::uint32_t padded_rows_amount = std::get<5>(marshalled_table_header.value()).value(); + + otable_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + if (!otable_witness) { + throw std::runtime_error("Something wrong with output witness_" + assignment_table_file_name); + } + otable_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + if (!otable_pub_inp) { + throw std::runtime_error("Something wrong with output pub_inp_" + assignment_table_file_name); + } + + print_witness(table_proxy, print_kind, padded_rows_amount, otable_witness); + print_pub_inp(table_proxy, print_kind, padded_rows_amount, otable_pub_inp); +} + +template +void print_assignment_table(const assignment_proxy &table_proxy, + print_table_kind print_kind, + std::uint32_t ComponentConstantColumns, std::uint32_t ComponentSelectorColumns, + std::string assignment_table_file_name, + std::uint32_t idx + ) { + + if (print_kind == print_table_kind::MULTI_PROVER) { + assignment_table_file_name = assignment_table_file_name + std::to_string(idx); + } + + std::ofstream otable_header; + std::ofstream otable_witness; + std::ofstream otable_pub_inp; + std::ofstream otable_constants; + std::ofstream otable_selectors; + otable_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + if (!otable_header) + throw std::runtime_error("Something wrong with output header_" + assignment_table_file_name); + otable_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + if (!otable_witness) + throw std::runtime_error("Something wrong with output witness_" + assignment_table_file_name); + otable_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + if (!otable_pub_inp) + throw std::runtime_error("Something wrong with output pub_inp_" + assignment_table_file_name); + otable_constants.open("constants_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + if (!otable_constants) + throw std::runtime_error("Something wrong with output constants_" + assignment_table_file_name); + otable_selectors.open("selectors_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + if (!otable_selectors) + throw std::runtime_error("Something wrong with output selectors_" + assignment_table_file_name); + + + using AssignmentTableType = assignment_proxy; + std::uint32_t usable_rows_amount; + std::uint32_t total_columns; + std::uint32_t total_size; + std::uint32_t shared_size = (print_kind == print_table_kind::MULTI_PROVER) ? 1 : 0; + std::uint32_t public_input_size = table_proxy.public_inputs_amount(); + std::uint32_t witness_size = table_proxy.witnesses_amount(); + std::uint32_t constant_size = table_proxy.constants_amount(); + std::uint32_t selector_size = table_proxy.selectors_amount(); + const auto lookup_constant_cols = table_proxy.get_lookup_constant_cols(); + const auto lookup_selector_cols = table_proxy.get_lookup_selector_cols(); + + std::uint32_t max_public_inputs_size = 0; + std::uint32_t max_constant_size = 0; + std::uint32_t max_selector_size = 0; + + for (std::uint32_t i = 0; i < public_input_size; i++) { + max_public_inputs_size = std::max(max_public_inputs_size, table_proxy.public_input_column_size(i)); + } + + if (print_kind == print_table_kind::MULTI_PROVER) { + total_columns = witness_size + shared_size + public_input_size + constant_size + selector_size; + std::uint32_t max_shared_size = 0; + for (std::uint32_t i = 0; i < shared_size; i++) { + max_shared_size = std::max(max_shared_size, table_proxy.shared_column_size(i)); } - ASSERT_MSG(witness_idx + pub_inp_idx + constant_idx + selector_idx == total_size, "Printed index not equal required assignment size" ); + for (const auto &i : lookup_constant_cols) { + max_constant_size = std::max(max_constant_size, table_proxy.constant_column_size(i)); + } + for (const auto &i : lookup_selector_cols) { + max_selector_size = std::max(max_selector_size, table_proxy.selector_column_size(i)); + } + usable_rows_amount = table_proxy.get_used_rows().size(); + usable_rows_amount = std::max({usable_rows_amount, max_shared_size, max_public_inputs_size, max_constant_size, max_selector_size}); + } else { // SINGLE_PROVER + total_columns = witness_size + shared_size + public_input_size + constant_size + selector_size; + std::uint32_t max_witness_size = 0; + for (std::uint32_t i = 0; i < witness_size; i++) { + max_witness_size = std::max(max_witness_size, table_proxy.witness_column_size(i)); + } + for (std::uint32_t i = 0; i < constant_size; i++) { + max_constant_size = std::max(max_constant_size, table_proxy.constant_column_size(i)); + } + for (std::uint32_t i = 0; i < selector_size; i++) { + max_selector_size = std::max(max_selector_size, table_proxy.selector_column_size(i)); + } + usable_rows_amount = std::max({max_witness_size, max_public_inputs_size, max_constant_size, max_selector_size}); + } + + std::uint32_t padded_rows_amount = std::pow(2, std::ceil(std::log2(usable_rows_amount))); + if (padded_rows_amount == usable_rows_amount) { + padded_rows_amount *= 2; + } + if (padded_rows_amount < 8) { + padded_rows_amount = 8; } + + print_size_t(witness_size, otable_header); + print_size_t(public_input_size + shared_size, otable_header); + print_size_t(constant_size, otable_header); + print_size_t(selector_size, otable_header); + print_size_t(usable_rows_amount, otable_header); + print_size_t(padded_rows_amount, otable_header); + + print_witness(table_proxy, print_kind, padded_rows_amount, otable_witness); + print_pub_inp(table_proxy, print_kind, padded_rows_amount, otable_pub_inp); + print_constants(table_proxy, print_kind, padded_rows_amount, ComponentConstantColumns, otable_constants); + print_selectors(table_proxy, print_kind, padded_rows_amount, ComponentSelectorColumns, otable_selectors); + + /* + TODO: maybe save this check in some form + ASSERT_MSG(witness_idx + pub_inp_idx + constant_idx + selector_idx == total_size, "Printed index not equal required assignment size" ); + */ } bool read_json( @@ -467,18 +588,14 @@ void assignment_table_printer( const std::size_t &ComponentConstantColumns, const std::size_t &ComponentSelectorColumns ) { - std::ofstream otable; - otable.open(assignment_table_file_name + std::to_string(idx), - std::ios_base::binary | std::ios_base::out); - if (!otable) { - throw std::runtime_error("Failed to open file: " + assignment_table_file_name + std::to_string(idx)); - } - print_assignment_table( - assigner_instance.assignments[idx], print_table_kind::MULTI_PROVER, ComponentConstantColumns, - ComponentSelectorColumns, otable); - - otable.close(); + assigner_instance.assignments[idx], + print_table_kind::MULTI_PROVER, + ComponentConstantColumns, + ComponentSelectorColumns, + assignment_table_file_name, + idx + ); } template @@ -521,6 +638,19 @@ int curve_dependent_main(std::string bytecode_file_name, crypto3::zk::snark::plonk_constraint_system; using AssignmentTableType = zk::snark::plonk_table>; + using Endianness = nil::marshalling::option::big_endian; + using TTypeBase = nil::marshalling::field_type; + using constant_column_marshalling_type = + nil::marshalling::types::array_list< + TTypeBase, + nil::crypto3::marshalling::types::field_element< + TTypeBase, + typename BlueprintFieldType::value_type>, + nil::marshalling::option::sequence_size_field_prefix< + nil::marshalling::types::integral + > + >; + boost::json::value public_input_json_value; if(public_input_file_name.empty()) { public_input_json_value = boost::json::array(); @@ -584,11 +714,56 @@ int curve_dependent_main(std::string bytecode_file_name, auto parse_ir_file_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - parse_ir_file_start); BOOST_LOG_TRIVIAL(debug) << "parse_ir_file_duration: " << parse_ir_file_duration.count() << "ms"; + + std::vector>> all_constant_columns; + + if (gen_mode == nil::blueprint::generation_mode::fast_tbl()) { + // parse constant columns + + for (std::size_t i = 0; i < max_num_provers; i++) { + std::ifstream iconstants; + std::string table_file_name = max_num_provers == 0 ? assignment_table_file_name : assignment_table_file_name + std::to_string(i); + + iconstants.open("constants_" + table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iconstants) { + std::cout << "Cannot open " << "constants_" + table_file_name << std::endl; + return 1; + } + std::vector v; + std::vector consant_cols_vector; + iconstants.seekg(0, std::ios_base::end); + const auto fsize = iconstants.tellg(); + consant_cols_vector.resize(fsize); + iconstants.seekg(0, std::ios_base::beg); + iconstants.read(reinterpret_cast(consant_cols_vector.data()), fsize); + if (!iconstants) { + std::cout << "Cannot parse input file " << assignment_table_file_name << std::endl; + return 1; + } + iconstants.close(); + constant_column_marshalling_type marshalled_constant_column_data; + auto citer = consant_cols_vector.begin(); + auto const_column_marshalling_status = marshalled_constant_column_data.read(citer, consant_cols_vector.size()); + + std::vector> constant_columns = + nil::crypto3::marshalling::types::make_field_element_columns_vector + ( + marshalled_constant_column_data, + // we must extract desc from table_header, but for now its ok to use default values + desc.constant_columns, // take from header, now it is + desc.rows_amount // take from header, now it is zero TODO + ); + all_constant_columns.emplace_back(constant_columns); + } + } + + auto parser_evaluation_start = std::chrono::high_resolution_clock::now(); if ( !assigner_instance.evaluate( public_input_json_value.as_array(), private_input_json_value.as_array(), + all_constant_columns, table_pieces ) ) { @@ -657,18 +832,15 @@ int curve_dependent_main(std::string bytecode_file_name, auto single_table_print_start = std::chrono::high_resolution_clock::now(); // print assignment table if (gen_mode.has_assignments()) { - std::ofstream otable; - otable.open(assignment_table_file_name, std::ios_base::binary | std::ios_base::out); - if (!otable) { - std::cout << "Something wrong with output " << assignment_table_file_name << std::endl; - return 1; - } print_assignment_table( - assigner_instance.assignments[0], print_table_kind::SINGLE_PROVER, ComponentConstantColumns, - ComponentSelectorColumns, otable); - - otable.close(); + assigner_instance.assignments[0], + print_table_kind::SINGLE_PROVER, + ComponentConstantColumns, + ComponentSelectorColumns, + assignment_table_file_name, + 0 + ); auto single_table_print_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - single_table_print_start); BOOST_LOG_TRIVIAL(debug) << "single_table_print_duration: " << single_table_print_duration.count() << "ms"; } From 442d9c83c57722cccced71f4b2e93bdb25882499 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 16 May 2024 14:09:07 +0000 Subject: [PATCH 08/54] update transpiler to read different column types from different files --- bin/transpiler/src/main.cpp | 73 ++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/bin/transpiler/src/main.cpp b/bin/transpiler/src/main.cpp index 41380f2c1..ed6b73e8e 100644 --- a/bin/transpiler/src/main.cpp +++ b/bin/transpiler/src/main.cpp @@ -361,23 +361,70 @@ int curve_dependent_main( AssignmentTableType assignment_table; { - std::ifstream iassignment; - iassignment.open(assignment_table_file_name, std::ios_base::binary | std::ios_base::in); - if (!iassignment) { - std::cout << "Cannot open " << assignment_table_file_name << std::endl; + std::ifstream iassignment_header; + std::ifstream iassignment_witness; + std::ifstream iassignment_pub_inp; + std::ifstream iassignment_constants; + std::ifstream iassignment_selectors; + + iassignment_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_header) { + std::cout << "Cannot open header_" << assignment_table_file_name << std::endl; return 1; } - std::vector v; - iassignment.seekg(0, std::ios_base::end); - const auto fsize = iassignment.tellg(); - v.resize(fsize); - iassignment.seekg(0, std::ios_base::beg); - iassignment.read(reinterpret_cast(v.data()), fsize); - if (!iassignment) { - std::cout << "Cannot parse input file " << assignment_table_file_name << std::endl; + iassignment_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_witness) { + std::cout << "Cannot open witness_" << assignment_table_file_name << std::endl; + return 1; + } + iassignment_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_pub_inp) { + std::cout << "Cannot open pub_inp_" << assignment_table_file_name << std::endl; + return 1; + } + iassignment_constants.open("constants_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_constants) { + std::cout << "Cannot open constants_" << assignment_table_file_name << std::endl; + return 1; + } + iassignment_selectors.open("selectors_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_selectors) { + std::cout << "Cannot open selectors_" << assignment_table_file_name << std::endl; return 1; } - iassignment.close(); + std::vector v; + iassignment_header.seekg(0, std::ios_base::end); + iassignment_witness.seekg(0, std::ios_base::end); + iassignment_pub_inp.seekg(0, std::ios_base::end); + iassignment_constants.seekg(0, std::ios_base::end); + iassignment_selectors.seekg(0, std::ios_base::end); + + const auto header_size = iassignment_header.tellg(); + const auto w_size = iassignment_witness.tellg(); + const auto pi_size = iassignment_pub_inp.tellg(); + const auto c_size = iassignment_constants.tellg(); + const auto s_size = iassignment_selectors.tellg(); + + v.resize(header_size + w_size + pi_size + c_size + s_size); + + iassignment_header.seekg(0, std::ios_base::beg); + iassignment_witness.seekg(0, std::ios_base::beg); + iassignment_pub_inp.seekg(0, std::ios_base::beg); + iassignment_constants.seekg(0, std::ios_base::beg); + iassignment_selectors.seekg(0, std::ios_base::beg); + + iassignment_header.read(reinterpret_cast(v.data()), header_size); + iassignment_witness.read(reinterpret_cast(v.data()) + header_size, w_size); + iassignment_pub_inp.read(reinterpret_cast(v.data()) + header_size + w_size, pi_size); + iassignment_constants.read(reinterpret_cast(v.data()) + header_size + w_size + pi_size, c_size); + iassignment_selectors.read(reinterpret_cast(v.data()) + header_size + w_size + pi_size + c_size, s_size); + + iassignment_header.close(); + iassignment_witness.close(); + iassignment_pub_inp.close(); + iassignment_constants.close(); + iassignment_selectors.close(); + table_value_marshalling_type marshalled_table_data; auto read_iter = v.begin(); auto status = marshalled_table_data.read(read_iter, v.size()); From c0500d82163f4046ef314f98d89e01e5139f40ed Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 16 May 2024 14:22:51 +0000 Subject: [PATCH 09/54] update assigner --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index c1cad6c35..a5474e656 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit c1cad6c3541a7534ce3e85e6ec5b7002f67553dd +Subproject commit a5474e6568fa7f488a3aabaf9927c3d7c39c3cff From 1e4e1e1d6599ae027e1e3c99164b1855d3c97147 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 16 May 2024 14:27:46 +0000 Subject: [PATCH 10/54] update crypto3 (add table_header_type) --- libs/crypto3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/crypto3 b/libs/crypto3 index 1baa61944..860a89c02 160000 --- a/libs/crypto3 +++ b/libs/crypto3 @@ -1 +1 @@ -Subproject commit 1baa619449013b5fb2962b7edc59a14dbe2f59f2 +Subproject commit 860a89c02e3a7c097d50908c1ec903c4ffeb3e0e From 9f19c570e36b528ba84f90f874b55c2482c1629f Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 17 May 2024 11:00:47 +0000 Subject: [PATCH 11/54] fix selectors printing bug --- bin/assigner/src/main.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 7185ec7dd..f29398768 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -247,7 +247,6 @@ void print_selectors(const assignment_proxy &table_proxy, const auto& rows = table_proxy.get_used_rows(); const auto& selector_rows = table_proxy.get_used_selector_rows(); - print_size_t(selector_size * padded_rows_amount, out); std::uint32_t selector_idx = 0; for (std::uint32_t i = 0; i < ComponentSelectorColumns; i++) { const auto column_size = table_proxy.selector_column_size(i); From f7e3aafd0e8e6396b144a317bef4a55f28a9a44a Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 17 May 2024 11:03:02 +0000 Subject: [PATCH 12/54] temporarily disable vm.count(table-pieces) --- bin/assigner/src/main.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index f29398768..63903d516 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -1088,11 +1088,12 @@ int main(int argc, char *argv[]) { if (vm.count("table-pieces")) { table_pieces_file_name = vm["table-pieces"].as(); - } else { - std::cerr << "Invalid command line argument - table-pieces file name is not specified" << std::endl; - std::cout << options_desc << std::endl; - return 1; } + // else { + // std::cerr << "Invalid command line argument - table-pieces file name is not specified" << std::endl; + // std::cout << options_desc << std::endl; + // return 1; + // } if (vm.count("elliptic-curve-type")) { elliptic_curve = vm["elliptic-curve-type"].as(); From 5f6176a9429599ed3eda4c7b7ec51aedcc196700 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 17 May 2024 11:07:34 +0000 Subject: [PATCH 13/54] update assigner --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index a5474e656..3fa4833c1 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit a5474e6568fa7f488a3aabaf9927c3d7c39c3cff +Subproject commit 3fa4833c138a9894885bb2e94a4bbfa22767a8e2 From 64eaeecb08d774416f4a4d4517423d67edb6397e Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 17 May 2024 12:46:04 +0000 Subject: [PATCH 14/54] temporarily disable bls examples and tests --- examples/cpp/CMakeLists.txt | 10 +++++----- tests/cpp/CMakeLists.txt | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 29235f078..cbc21f3b1 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -189,8 +189,8 @@ add_example_with_proving(gate_argument_verifier SOURCES placeholder_prover/gate_ add_example_with_proving(permutation_argument_verifier SOURCES placeholder_prover/permutation_argument_verifier.cpp INPUT placeholder_prover/permutation_argument_verifier.inp CURVE_TYPE pallas) add_example_with_proving(lookup_argument_verifier SOURCES placeholder_prover/lookup_argument_verifier.cpp INPUT placeholder_prover/lookup_argument_verifier.inp CURVE_TYPE pallas) -add_example_without_proving(bls12_381_field_add_cpp SOURCES bls12_381_field_add.cpp INPUT bls12_381_field_add.inp CURVE_TYPE bls12381) -add_example_without_proving(bls12_381_field_mul_cpp SOURCES bls12_381_field_mul.cpp INPUT bls12_381_field_mul.inp CURVE_TYPE bls12381) -add_example_without_proving(bls12_381_field_sub_cpp SOURCES bls12_381_field_sub.cpp INPUT bls12_381_field_sub.inp CURVE_TYPE bls12381) -add_example_without_proving(bls12_381_signature_verification_cpp SOURCES bls12_381_signature_verification.cpp INPUT bls12_381_signature_verification.inp CURVE_TYPE bls12381) -add_example_without_proving(bls12_381_signature_verification_aggregated_cpp SOURCES bls12_381_signature_verification_aggregated.cpp INPUT bls12_381_signature_verification_aggregated.inp CURVE_TYPE bls12381) +# add_example_without_proving(bls12_381_field_add_cpp SOURCES bls12_381_field_add.cpp INPUT bls12_381_field_add.inp CURVE_TYPE bls12381) +# add_example_without_proving(bls12_381_field_mul_cpp SOURCES bls12_381_field_mul.cpp INPUT bls12_381_field_mul.inp CURVE_TYPE bls12381) +# add_example_without_proving(bls12_381_field_sub_cpp SOURCES bls12_381_field_sub.cpp INPUT bls12_381_field_sub.inp CURVE_TYPE bls12381) +# add_example_without_proving(bls12_381_signature_verification_cpp SOURCES bls12_381_signature_verification.cpp INPUT bls12_381_signature_verification.inp CURVE_TYPE bls12381) +# add_example_without_proving(bls12_381_signature_verification_aggregated_cpp SOURCES bls12_381_signature_verification_aggregated.cpp INPUT bls12_381_signature_verification_aggregated.inp CURVE_TYPE bls12381) diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 0402b3eee..be34d4d76 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -263,8 +263,8 @@ add_zkllvm_unit_test("algebra/recursive_prover/lookup_arg_verifier/test1/test") add_zkllvm_unit_test("algebra/recursive_prover/lookup_arg_verifier/test2/test") # bls tests -add_zkllvm_unit_test_bls12381("algebra/fields/bls12381_fp12_arithmetics/mul") -add_zkllvm_unit_test_bls12381("bls_signature/pairing/pairing") +# add_zkllvm_unit_test_bls12381("algebra/fields/bls12381_fp12_arithmetics/mul") +# add_zkllvm_unit_test_bls12381("bls_signature/pairing/pairing") add_zkllvm_unit_test("nested_if/first") add_zkllvm_unit_test("nested_if/second") From 6e7c13d3eefc97906e883439968a52c0e88dfdd3 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 13:03:29 +0000 Subject: [PATCH 15/54] update blueprint(fix typo) --- libs/blueprint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/blueprint b/libs/blueprint index b05f6c370..b822e1697 160000 --- a/libs/blueprint +++ b/libs/blueprint @@ -1 +1 @@ -Subproject commit b05f6c370b9301b14b4ce6109fefe6f1ddb96da1 +Subproject commit b822e1697a3713efb95646e428a4a26268be7922 From fbc83f2829143756b280d39b59fe0be4995363ba Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 13:50:53 +0000 Subject: [PATCH 16/54] update assigner (avoid info level logging) --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 3fa4833c1..ba10a2045 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 3fa4833c138a9894885bb2e94a4bbfa22767a8e2 +Subproject commit ba10a2045a1dd7d37331390350d0f18bfd913839 From 3d981531883bdc1199708e9c42746dbc4a83044b Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 14:17:36 +0000 Subject: [PATCH 17/54] update recursive_gen to read different columns from different files --- bin/recursive_gen/src/main.cpp | 90 +++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/bin/recursive_gen/src/main.cpp b/bin/recursive_gen/src/main.cpp index 645f0066c..c188d1283 100644 --- a/bin/recursive_gen/src/main.cpp +++ b/bin/recursive_gen/src/main.cpp @@ -106,6 +106,94 @@ std::optional decode_marshalling_from_file( return marshalled_data; } +template +std::optional decode_marshalling_from_different_column_types_files( + const std::string& assignment_table_file_name + //bool hex = false // Used only for placeholder proof +) { + //////// + std::ifstream iassignment_header; + std::ifstream iassignment_witness; + std::ifstream iassignment_pub_inp; + std::ifstream iassignment_constants; + std::ifstream iassignment_selectors; + + iassignment_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_header) { + std::cout << "Cannot open header_" << assignment_table_file_name << std::endl; + return std::nullopt; + } + iassignment_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_witness) { + std::cout << "Cannot open witness_" << assignment_table_file_name << std::endl; + return std::nullopt; + } + iassignment_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_pub_inp) { + std::cout << "Cannot open pub_inp_" << assignment_table_file_name << std::endl; + return std::nullopt; + } + iassignment_constants.open("constants_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_constants) { + std::cout << "Cannot open constants_" << assignment_table_file_name << std::endl; + return std::nullopt; + } + iassignment_selectors.open("selectors_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + if (!iassignment_selectors) { + std::cout << "Cannot open selectors_" << assignment_table_file_name << std::endl; + return std::nullopt; + } + std::vector v; + iassignment_header.seekg(0, std::ios_base::end); + iassignment_witness.seekg(0, std::ios_base::end); + iassignment_pub_inp.seekg(0, std::ios_base::end); + iassignment_constants.seekg(0, std::ios_base::end); + iassignment_selectors.seekg(0, std::ios_base::end); + + const auto header_size = iassignment_header.tellg(); + const auto w_size = iassignment_witness.tellg(); + const auto pi_size = iassignment_pub_inp.tellg(); + const auto c_size = iassignment_constants.tellg(); + const auto s_size = iassignment_selectors.tellg(); + + v.resize(header_size + w_size + pi_size + c_size + s_size); + + iassignment_header.seekg(0, std::ios_base::beg); + iassignment_witness.seekg(0, std::ios_base::beg); + iassignment_pub_inp.seekg(0, std::ios_base::beg); + iassignment_constants.seekg(0, std::ios_base::beg); + iassignment_selectors.seekg(0, std::ios_base::beg); + + iassignment_header.read(reinterpret_cast(v.data()), header_size); + iassignment_witness.read(reinterpret_cast(v.data()) + header_size, w_size); + iassignment_pub_inp.read(reinterpret_cast(v.data()) + header_size + w_size, pi_size); + iassignment_constants.read(reinterpret_cast(v.data()) + header_size + w_size + pi_size, c_size); + iassignment_selectors.read(reinterpret_cast(v.data()) + header_size + w_size + pi_size + c_size, s_size); + + iassignment_header.close(); + iassignment_witness.close(); + iassignment_pub_inp.close(); + iassignment_constants.close(); + iassignment_selectors.close(); + + MarshallingType marshalled_data; + auto read_iter = v.begin(); + auto status = marshalled_data.read(read_iter, v.size()); + + if (status != nil::marshalling::status_type::success) { + std::cerr << "Marshalled structure decoding failed" << std::endl; + return std::nullopt; + } + return marshalled_data; + + // std::tie(desc, assignment_table) = + // nil::crypto3::marshalling::types::make_assignment_table( + // marshalled_table_data + // ); + ///////// +} + + template struct ParametersPolicy { constexpr static const std::size_t WitnessColumns = WITNESS_COLUMNS; @@ -414,7 +502,7 @@ int curve_dependent_main( public_input_sizes = (*constraint_system).public_input_sizes(); if( vm.count("assignment-table") ){ - auto marshalled_value = decode_marshalling_from_file(assignment_table_file_name); + auto marshalled_value = decode_marshalling_from_different_column_types_files(assignment_table_file_name); if (!marshalled_value) { return false; } From 83173b35bad676df8e3910feb34a781e2459266d Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 15:26:52 +0000 Subject: [PATCH 18/54] assigner main.cpp handle output filename with subfolders --- bin/assigner/src/main.cpp | 42 +++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 63903d516..076ac2c6e 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -65,6 +65,7 @@ #include #include #include +#include using namespace nil; using namespace nil::crypto3; @@ -426,6 +427,20 @@ void print_assignment_table_fast( print_pub_inp(table_proxy, print_kind, padded_rows_amount, otable_pub_inp); } +std::string add_filename_prefix( + const std::string& prefix, + const std::string& file_name +) { + std::filesystem::path path(file_name); + std::filesystem::path parent_path = path.parent_path(); + std::filesystem::path filename = path.filename(); + + std::string new_filename = prefix + filename.string(); + std::filesystem::path new_path = parent_path / new_filename; + + return new_path.string(); +} + template void print_assignment_table(const assignment_proxy &table_proxy, print_table_kind print_kind, @@ -443,21 +458,22 @@ void print_assignment_table(const assignment_proxy &table_p std::ofstream otable_pub_inp; std::ofstream otable_constants; std::ofstream otable_selectors; - otable_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + + otable_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); if (!otable_header) - throw std::runtime_error("Something wrong with output header_" + assignment_table_file_name); - otable_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + throw std::runtime_error("Something wrong with output " + add_filename_prefix("header_", assignment_table_file_name)); + otable_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); if (!otable_witness) - throw std::runtime_error("Something wrong with output witness_" + assignment_table_file_name); - otable_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + throw std::runtime_error("Something wrong with output " + add_filename_prefix("witness_", assignment_table_file_name)); + otable_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); if (!otable_pub_inp) - throw std::runtime_error("Something wrong with output pub_inp_" + assignment_table_file_name); - otable_constants.open("constants_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + throw std::runtime_error("Something wrong with output " + add_filename_prefix("pub_inp_", assignment_table_file_name)); + otable_constants.open(add_filename_prefix("constants_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); if (!otable_constants) - throw std::runtime_error("Something wrong with output constants_" + assignment_table_file_name); - otable_selectors.open("selectors_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + throw std::runtime_error("Something wrong with output " + add_filename_prefix("constants_", assignment_table_file_name)); + otable_selectors.open(add_filename_prefix("selectors_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); if (!otable_selectors) - throw std::runtime_error("Something wrong with output selectors_" + assignment_table_file_name); + throw std::runtime_error("Something wrong with output " + add_filename_prefix("selectors_", assignment_table_file_name)); using AssignmentTableType = assignment_proxy; @@ -868,6 +884,12 @@ int curve_dependent_main(std::string bytecode_file_name, // print assignment table if (gen_mode.has_assignments()) { + // print_table_human_readable( + // assigner_instance.assignments[idx], + // ComponentConstantColumns, + // ComponentSelectorColumns + // ); + auto future = std::async( std::launch::async, assignment_table_printer, From cc2745df6edfa0cf7b7274726c36e14cd07adb52 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 15:32:10 +0000 Subject: [PATCH 19/54] transpiler main.cpp handle output filename with subfolders --- bin/transpiler/src/main.cpp | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/bin/transpiler/src/main.cpp b/bin/transpiler/src/main.cpp index ed6b73e8e..65a0e6258 100644 --- a/bin/transpiler/src/main.cpp +++ b/bin/transpiler/src/main.cpp @@ -242,6 +242,20 @@ int main(int argc, char *argv[]) { } +std::string add_filename_prefix( + const std::string& prefix, + const std::string& file_name +) { + std::filesystem::path path(file_name); + std::filesystem::path parent_path = path.parent_path(); + std::filesystem::path filename = path.filename(); + + std::string new_filename = prefix + filename.string(); + std::filesystem::path new_path = parent_path / new_filename; + + return new_path.string(); +} + template int curve_dependent_main( boost::program_options::options_description options_desc, @@ -367,29 +381,29 @@ int curve_dependent_main( std::ifstream iassignment_constants; std::ifstream iassignment_selectors; - iassignment_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_header) { - std::cout << "Cannot open header_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("header_", assignment_table_file_name) << std::endl; return 1; } - iassignment_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_witness) { - std::cout << "Cannot open witness_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("witness_", assignment_table_file_name) << std::endl; return 1; } - iassignment_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_pub_inp) { - std::cout << "Cannot open pub_inp_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("pub_inp_", assignment_table_file_name) << std::endl; return 1; } - iassignment_constants.open("constants_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_constants.open(add_filename_prefix("constants_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_constants) { - std::cout << "Cannot open constants_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("constants_", assignment_table_file_name) << std::endl; return 1; } - iassignment_selectors.open("selectors_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_selectors.open(add_filename_prefix("selectors_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_selectors) { - std::cout << "Cannot open selectors_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("selectors_", assignment_table_file_name) << std::endl; return 1; } std::vector v; From a269eea4de46ab19cd9c5b0ff55fcf425f0ae8c0 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 15:46:00 +0000 Subject: [PATCH 20/54] recursive_gen main.cpp handle output filename with subfolders --- bin/recursive_gen/src/main.cpp | 41 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/bin/recursive_gen/src/main.cpp b/bin/recursive_gen/src/main.cpp index c188d1283..051678ab5 100644 --- a/bin/recursive_gen/src/main.cpp +++ b/bin/recursive_gen/src/main.cpp @@ -106,41 +106,54 @@ std::optional decode_marshalling_from_file( return marshalled_data; } +std::string add_filename_prefix( + const std::string& prefix, + const std::string& file_name +) { + std::filesystem::path path(file_name); + std::filesystem::path parent_path = path.parent_path(); + std::filesystem::path filename = path.filename(); + + std::string new_filename = prefix + filename.string(); + std::filesystem::path new_path = parent_path / new_filename; + + return new_path.string(); +} + template std::optional decode_marshalling_from_different_column_types_files( const std::string& assignment_table_file_name //bool hex = false // Used only for placeholder proof ) { - //////// std::ifstream iassignment_header; std::ifstream iassignment_witness; std::ifstream iassignment_pub_inp; std::ifstream iassignment_constants; std::ifstream iassignment_selectors; - iassignment_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_header) { - std::cout << "Cannot open header_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("header_", assignment_table_file_name) << std::endl; return std::nullopt; } - iassignment_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_witness) { - std::cout << "Cannot open witness_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("witness_", assignment_table_file_name) << std::endl; return std::nullopt; } - iassignment_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_pub_inp) { - std::cout << "Cannot open pub_inp_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("pub_inp_", assignment_table_file_name) << std::endl; return std::nullopt; } - iassignment_constants.open("constants_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_constants.open(add_filename_prefix("constants_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_constants) { - std::cout << "Cannot open constants_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("constants_", assignment_table_file_name) << std::endl; return std::nullopt; } - iassignment_selectors.open("selectors_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + iassignment_selectors.open(add_filename_prefix("selectors_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!iassignment_selectors) { - std::cout << "Cannot open selectors_" << assignment_table_file_name << std::endl; + std::cout << "Cannot open " << add_filename_prefix("selectors_", assignment_table_file_name) << std::endl; return std::nullopt; } std::vector v; @@ -185,12 +198,6 @@ std::optional decode_marshalling_from_different_column_types_fi return std::nullopt; } return marshalled_data; - - // std::tie(desc, assignment_table) = - // nil::crypto3::marshalling::types::make_assignment_table( - // marshalled_table_data - // ); - ///////// } From 6af1e94d84e767f86e9d1f626f1fb6e1e99ab0a5 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 15:49:32 +0000 Subject: [PATCH 21/54] update crypto3 (add boost json header to zk variable) --- libs/crypto3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/crypto3 b/libs/crypto3 index 860a89c02..daf189cfd 160000 --- a/libs/crypto3 +++ b/libs/crypto3 @@ -1 +1 @@ -Subproject commit 860a89c02e3a7c097d50908c1ec903c4ffeb3e0e +Subproject commit daf189cfd4c5baed27e555f1dabc579743be1f11 From 569c97309622063d70fadeef204413189b89ba09 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 21 May 2024 16:46:38 +0000 Subject: [PATCH 22/54] move add_filename_prefix function --- bin/assigner/src/main.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 076ac2c6e..3f7512f9c 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -386,6 +386,20 @@ void print_witness(const assignment_proxy &table_proxy, } } +std::string add_filename_prefix( + const std::string& prefix, + const std::string& file_name +) { + std::filesystem::path path(file_name); + std::filesystem::path parent_path = path.parent_path(); + std::filesystem::path filename = path.filename(); + + std::string new_filename = prefix + filename.string(); + std::filesystem::path new_path = parent_path / new_filename; + + return new_path.string(); +} + template void print_assignment_table_fast( const assignment_proxy &table_proxy, @@ -427,20 +441,6 @@ void print_assignment_table_fast( print_pub_inp(table_proxy, print_kind, padded_rows_amount, otable_pub_inp); } -std::string add_filename_prefix( - const std::string& prefix, - const std::string& file_name -) { - std::filesystem::path path(file_name); - std::filesystem::path parent_path = path.parent_path(); - std::filesystem::path filename = path.filename(); - - std::string new_filename = prefix + filename.string(); - std::filesystem::path new_path = parent_path / new_filename; - - return new_path.string(); -} - template void print_assignment_table(const assignment_proxy &table_proxy, print_table_kind print_kind, From 5f42f66f53bf92043aa02b024ee7cbe68fb909cb Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 22 May 2024 22:38:38 +0000 Subject: [PATCH 23/54] fix generate_type for fast tbl case --- bin/assigner/src/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 3f7512f9c..00b977a7d 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -1053,7 +1053,7 @@ int main(int argc, char *argv[]) { } else if (generate_type == "assignment") { gen_mode = nil::blueprint::generation_mode::assignments(); } else if (generate_type == "assignment-fast") { - gen_mode = nil::blueprint::generation_mode::fast_tbl() | nil::blueprint::generation_mode::assignments(); + gen_mode = nil::blueprint::generation_mode::fast_tbl(); } else if (generate_type == "size_estimation") { gen_mode = nil::blueprint::generation_mode::size_estimation(); } else if (generate_type == "public-input-column") { @@ -1088,7 +1088,7 @@ int main(int argc, char *argv[]) { public_input_file_name = vm["public-input"].as(); } - if (gen_mode.has_assignments()) { + if (gen_mode.has_assignments() || gen_mode.has_fast_tbl()) { if (vm.count("assignment-table")) { assignment_table_file_name = vm["assignment-table"].as(); } else { From 0bc72675a740bc8724d21b6f8bf63df2dad4fbb9 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 22 May 2024 22:52:10 +0000 Subject: [PATCH 24/54] add fast tbl printer that works --- bin/assigner/src/main.cpp | 84 ++++++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 00b977a7d..490abde34 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -404,13 +404,18 @@ template &table_proxy, print_table_kind print_kind, - std::string assignment_table_file_name + std::string assignment_table_file_name, + std::uint32_t idx ) { + if (print_kind == print_table_kind::MULTI_PROVER) { + assignment_table_file_name = assignment_table_file_name + std::to_string(idx); + } + std::ifstream itable_header; std::ofstream otable_witness; std::ofstream otable_pub_inp; - itable_header.open("header_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::in); + itable_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!itable_header) { throw std::runtime_error("Something wrong with input file header_" + assignment_table_file_name); // TODO: add catch } @@ -422,19 +427,22 @@ void print_assignment_table_fast( itable_header.seekg(0, std::ios_base::end); const auto header_size = itable_header.tellg(); header_binary.resize(header_size); + itable_header.seekg(0, std::ios_base::beg); + itable_header.read(reinterpret_cast(header_binary.data()), header_size); + itable_header.close(); auto hiter = header_binary.begin(); auto status = marshalled_table_header.read(hiter, header_binary.size()); -// TODO add status check + ASSERT(status == nil::marshalling::status_type::success); std::uint32_t padded_rows_amount = std::get<5>(marshalled_table_header.value()).value(); - otable_witness.open("witness_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + otable_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); if (!otable_witness) { - throw std::runtime_error("Something wrong with output witness_" + assignment_table_file_name); + throw std::runtime_error("Something wrong with output " + add_filename_prefix("witness_", assignment_table_file_name)); } - otable_pub_inp.open("pub_inp_" + assignment_table_file_name, std::ios_base::binary | std::ios_base::out); + otable_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); if (!otable_pub_inp) { - throw std::runtime_error("Something wrong with output pub_inp_" + assignment_table_file_name); + throw std::runtime_error("Something wrong with output " + add_filename_prefix("pub_inp_", assignment_table_file_name)); } print_witness(table_proxy, print_kind, padded_rows_amount, otable_witness); @@ -732,7 +740,7 @@ int curve_dependent_main(std::string bytecode_file_name, std::vector>> all_constant_columns; - if (gen_mode == nil::blueprint::generation_mode::fast_tbl()) { + if (gen_mode.has_fast_tbl()) { // parse constant columns for (std::size_t i = 0; i < max_num_provers; i++) { @@ -759,14 +767,40 @@ int curve_dependent_main(std::string bytecode_file_name, constant_column_marshalling_type marshalled_constant_column_data; auto citer = consant_cols_vector.begin(); auto const_column_marshalling_status = marshalled_constant_column_data.read(citer, consant_cols_vector.size()); +/// + std::ifstream itable_header; + std::ofstream otable_witness; + std::ofstream otable_pub_inp; + itable_header.open(add_filename_prefix("header_", table_file_name), std::ios_base::in); + if (!itable_header) { + throw std::runtime_error("Something wrong with input file header_" + table_file_name); // TODO: add catch + } + + using TTypeBase = nil::marshalling::field_type; + using table_header_marshalling_type = nil::crypto3::marshalling::types::table_header_type; + table_header_marshalling_type marshalled_table_header; + std::vector header_binary; + itable_header.seekg(0, std::ios_base::end); + const auto header_size = itable_header.tellg(); + header_binary.resize(header_size); + itable_header.seekg(0, std::ios_base::beg); + itable_header.read(reinterpret_cast(header_binary.data()), header_size); + itable_header.close(); + auto hiter = header_binary.begin(); + auto status = marshalled_table_header.read(hiter, header_binary.size()); + ASSERT(status == nil::marshalling::status_type::success); + + std::uint32_t columns_rows_amount = std::get<5>(marshalled_table_header.value()).value(); + +/// std::vector> constant_columns = nil::crypto3::marshalling::types::make_field_element_columns_vector ( marshalled_constant_column_data, // we must extract desc from table_header, but for now its ok to use default values - desc.constant_columns, // take from header, now it is - desc.rows_amount // take from header, now it is zero TODO + desc.constant_columns, // TODO we should take it from header file, but it will almost always work okay without it + columns_rows_amount ); all_constant_columns.emplace_back(constant_columns); } @@ -807,6 +841,36 @@ int curve_dependent_main(std::string bytecode_file_name, file.close(); } + if (gen_mode.has_fast_tbl()) { + if (assigner_instance.assignments.size() == 1) { + auto fast_tbl_print_start = std::chrono::high_resolution_clock::now(); + print_assignment_table_fast( + assigner_instance.assignments[0], + print_table_kind::SINGLE_PROVER, + assignment_table_file_name, + 0 + ); + auto fast_tbl_print_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - fast_tbl_print_start); + BOOST_LOG_TRIVIAL(debug) << "fast_tbl_print_duration: " << fast_tbl_print_duration.count() << "ms"; + return 0; + } + else { + auto fast_tbl_print_start = std::chrono::high_resolution_clock::now(); + // print assignment table + for (std::size_t i = 0; i < assigner_instance.assignments.size(); i++) { + print_assignment_table_fast( + assigner_instance.assignments[i], + print_table_kind::MULTI_PROVER, + assignment_table_file_name, + i + ); + } + auto fast_tbl_print_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - fast_tbl_print_start); + BOOST_LOG_TRIVIAL(debug) << "fast_tbl_print_duration: " << fast_tbl_print_duration.count() << "ms"; + return 0; + } + } + auto pack_lookup_start = std::chrono::high_resolution_clock::now(); // pack lookup tables if (assigner_instance.circuits[0].get_reserved_tables().size() > 0) { From dba02ecd0044dac57eddd1bf899dbc042a159009 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 23 May 2024 17:58:26 +0000 Subject: [PATCH 25/54] add used_rows and to_be_shared reading/writing --- bin/assigner/src/main.cpp | 60 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 490abde34..a92ee890b 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -689,7 +689,12 @@ int curve_dependent_main(std::string bytecode_file_name, } using var = crypto3::zk::snark::plonk_variable; + std::vector> table_pieces = {}; // we create table pieces in any case and pass into assigner by reference + std::vector> all_used_rows; + int provers_amount; + std::vector> to_be_shared; + if (gen_mode.has_fast_tbl()) { // if we are generating tables in a fast way then need to parse table_pieces from file std::ifstream inp_json(table_pieces_file_name); @@ -702,11 +707,31 @@ int curve_dependent_main(std::string bytecode_file_name, std::string str((std::istreambuf_iterator(inp_json)), std::istreambuf_iterator()); auto parsed = boost::json::parse(str); - boost::json::array arr = parsed.as_array(); + boost::json::object obj = parsed.as_object(); + boost::json::array arr = obj["table_pieces"].as_array(); for (const auto& item : arr) { table_pieces.emplace_back(item.as_object()); } + + provers_amount = obj["provers_amount"].as_int64(); + + boost::json::array used_rows_json = obj["used_rows"].as_array(); + for (const auto& row : used_rows_json) { + std::vector current_used_rows = {}; + for (const auto& elem : row.as_array()) { + current_used_rows.push_back(elem.as_int64()); + } + all_used_rows.push_back(current_used_rows); + } + + boost::json::array to_be_shared_json = obj["to_be_shared"].as_array(); + for (const auto& pair_json : to_be_shared_json) { + boost::json::object pair_obj = pair_json.as_object(); + std::uint32_t first = pair_obj["first"].as_int64(); + var second = var(pair_obj["second"].as_object()); + to_be_shared.emplace_back(first, second); + } } auto assigner_instance_creation_start = std::chrono::high_resolution_clock::now(); @@ -743,9 +768,9 @@ int curve_dependent_main(std::string bytecode_file_name, if (gen_mode.has_fast_tbl()) { // parse constant columns - for (std::size_t i = 0; i < max_num_provers; i++) { + for (std::size_t i = 0; i < provers_amount; i++) { std::ifstream iconstants; - std::string table_file_name = max_num_provers == 0 ? assignment_table_file_name : assignment_table_file_name + std::to_string(i); + std::string table_file_name = provers_amount == 1 ? assignment_table_file_name : assignment_table_file_name + std::to_string(i); iconstants.open("constants_" + table_file_name, std::ios_base::binary | std::ios_base::in); if (!iconstants) { @@ -813,6 +838,8 @@ int curve_dependent_main(std::string bytecode_file_name, public_input_json_value.as_array(), private_input_json_value.as_array(), all_constant_columns, + all_used_rows, + to_be_shared, table_pieces ) ) { @@ -828,13 +855,38 @@ int curve_dependent_main(std::string bytecode_file_name, } if (gen_mode.has_circuit()) { // if we are generation circuit then we are generation table pieces in the same time. Need to write itno file + + boost::json::object top_level_json; + top_level_json["provers_amount"] = assigner_instance.assignments.size(); + boost::json::array pieces_json; // for (const auto& piece : nil::blueprint::table_pieces) { for (const auto& piece : nil::blueprint::table_pieces) { pieces_json.push_back(piece.to_json()); } + top_level_json["table_pieces"] = pieces_json; + + boost::json::array used_rows_json; + for (std::size_t i = 0; i < assigner_instance.assignments.size(); i++) { + boost::json::array current_row; + for (const auto& r : assigner_instance.assignments[i].get_used_rows()) { + current_row.push_back(r); + } + used_rows_json.push_back(current_row); + } + top_level_json["used_rows"] = used_rows_json; + + boost::json::array to_be_shared_json; + + for (const auto& pair : nil::blueprint::to_be_shared) { + boost::json::object pair_json; + pair_json["first"] = pair.first; + pair_json["second"] = pair.second.to_json(); + to_be_shared_json.push_back(pair_json); + } + top_level_json["to_be_shared"] = to_be_shared_json; - std::string serialized = boost::json::serialize(pieces_json); + std::string serialized = boost::json::serialize(top_level_json); std::ofstream file(table_pieces_file_name); file << serialized; From e339f87225ffbe09c4dc2f0c8ae2c5dabf887628 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 23 May 2024 18:07:20 +0000 Subject: [PATCH 26/54] update assigner (used_rows and to_be_shared handling) --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index ba10a2045..677797e27 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit ba10a2045a1dd7d37331390350d0f18bfd913839 +Subproject commit 677797e27a44c2b80cf79c5d01aa76edf6a52925 From c17b69f859987cbec1d30ee9a110121a0f1d3918 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 28 May 2024 10:54:37 +0000 Subject: [PATCH 27/54] update assigner --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 677797e27..0b85d5f27 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 677797e27a44c2b80cf79c5d01aa76edf6a52925 +Subproject commit 0b85d5f27eae691131227c2160ec7728cd0826c2 From d7f13b1fbc41af96f6e2147a2e85d3186a575432 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 28 May 2024 12:20:34 +0000 Subject: [PATCH 28/54] add fast tbl generation in CMakeLists gen_proof --- examples/CMakeLists.txt | 15 ++++++++++----- examples/cpp/CMakeLists.txt | 20 ++++++++++++++------ examples/rust/CMakeLists.txt | 8 ++++++-- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index f48e326eb..2450883c0 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -137,15 +137,15 @@ function(assign_ir) VERBATIM) endfunction() -function(gen_proof target curve_type provers_amount) +function(gen_proof target curve_type provers_amount tbl_gen_speed) if(provers_amount EQUAL 0) - gen_single_proof(${target} ${curve_type} 0) + gen_single_proof(${target} ${curve_type} 0 ${tbl_gen_speed}) else() add_custom_target(${target}_prove) foreach(prover_num RANGE 1 ${provers_amount}) math(EXPR prover_num_minus_1 "${prover_num} - 1") - gen_single_proof(${target} ${curve_type} ${prover_num}) + gen_single_proof(${target} ${curve_type} ${prover_num} ${tbl_gen_speed}) add_dependencies(${target}_prove ${target}_prove${prover_num_minus_1}) endforeach() @@ -160,7 +160,7 @@ function(gen_proof target curve_type provers_amount) endfunction() -function(gen_single_proof target curve_type provers_amount) +function(gen_single_proof target curve_type provers_amount tbl_gen_speed) if(NOT provers_amount EQUAL 0) set(multi_prover_flag --multi-prover) math(EXPR prover_num "${provers_amount} - 1") @@ -168,6 +168,11 @@ function(gen_single_proof target curve_type provers_amount) set(prover_num "") endif() + set (depends_tbl ${target}_generate_tbl) + if (tbl_gen_speed STREQUAL "fast") + set (depends_tbl ${target}_generate_tbl_fast) + endif() + add_custom_target(${target}_prove${prover_num} COMMAND $ -m gen-test-proof -c circuit_${target}.crct${prover_num} @@ -175,7 +180,7 @@ function(gen_single_proof target curve_type provers_amount) -o transpiler_output_${target}${prover_num} -e ${curve_type} ${multi_prover_flag} - DEPENDS ${target}_generate_crct ${target}_generate_tbl $ + DEPENDS ${target}_generate_crct ${depends_tbl} $ COMMAND_EXPAND_LISTS VERBATIM) endfunction() diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index cbc21f3b1..099188f65 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -80,7 +80,7 @@ endfunction() function(add_example_with_proving example_target) set(prefix ARG) set(noValues "") - set(singleValues INPUT PRIVATE_INPUT CURVE_TYPE MAX_NUM_PROVERS) + set(singleValues INPUT PRIVATE_INPUT CURVE_TYPE MAX_NUM_PROVERS TBL_GEN_SPEED) set(multiValues SOURCES) cmake_parse_arguments(${prefix} "${noValues}" @@ -94,7 +94,11 @@ function(add_example_with_proving example_target) set(ARG_MAX_NUM_PROVERS 0) endif() - gen_proof(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS}) + if(NOT DEFINED ARG_TBL_GEN_SPEED) + set(ARG_TBL_GEN_SPEED "slow") + endif() + + gen_proof(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS} ${ARG_TBL_GEN_SPEED}) add_dependencies(prove_cpp_examples ${example_target}_prove) if(GENERATE_EVM_VERIFIER) @@ -107,7 +111,7 @@ endfunction() function(add_example_with_proving_and_recursion example_target) set(prefix ARG) set(noValues "") - set(singleValues INPUT PRIVATE_INPUT CURVE_TYPE MAX_NUM_PROVERS) + set(singleValues INPUT PRIVATE_INPUT CURVE_TYPE TBL_GEN_SPEED MAX_NUM_PROVERS) set(multiValues SOURCES) cmake_parse_arguments(${prefix} "${noValues}" @@ -115,12 +119,16 @@ function(add_example_with_proving_and_recursion example_target) "${multiValues}" ${ARGN}) - add_example_with_proving(${example_target} ${ARG_CURVE_TYPE} ${ARGN}) - if(NOT DEFINED ARG_MAX_NUM_PROVERS) set(ARG_MAX_NUM_PROVERS 0) endif() + if(NOT DEFINED ARG_TBL_GEN_SPEED) + set(ARG_TBL_GEN_SPEED "slow") + endif() + + add_example_with_proving(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS} ${ARG_TBL_GEN_SPEED} ${ARGN}) + add_custom_target(${example_target}_recursion) assign_recursive_verifiers(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS}) add_dependencies(${example_target}_recursion ${example_target}_assign_placeholder_verifiers) @@ -129,7 +137,7 @@ endfunction() add_custom_target("recursion") -add_example_with_proving(arithmetics_cpp_example SOURCES arithmetics.cpp INPUT arithmetics.inp CURVE_TYPE pallas) +add_example_with_proving(arithmetics_cpp_example SOURCES arithmetics.cpp INPUT arithmetics.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(integer_arithmetics_cpp_example SOURCES integer_arithmetics.cpp INPUT integer_arithmetics.inp CURVE_TYPE pallas) add_example_with_proving(poseidon_cpp_example SOURCES poseidon.cpp INPUT poseidon.inp CURVE_TYPE pallas) add_example_with_proving_and_recursion(merkle_tree_poseidon_cpp_example SOURCES merkle_tree_poseidon.cpp INPUT merkle_tree_poseidon.inp CURVE_TYPE pallas) diff --git a/examples/rust/CMakeLists.txt b/examples/rust/CMakeLists.txt index 0021a40b0..573610924 100644 --- a/examples/rust/CMakeLists.txt +++ b/examples/rust/CMakeLists.txt @@ -58,7 +58,7 @@ endfunction() function(add_rust_example_with_proving example_target) set(prefix ARG) set(noValues "") - set(singleValues EXAMPLE_NAME INPUT CURVE_TYPE) + set(singleValues EXAMPLE_NAME INPUT CURVE_TYPE TBL_GEN_SPEED) set(multiValues) cmake_parse_arguments(${prefix} "${noValues}" @@ -72,7 +72,11 @@ function(add_rust_example_with_proving example_target) set(ARG_MAX_NUM_PROVERS 0) endif() - gen_proof(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS}) + if(NOT DEFINED ARG_TBL_GEN_SPEED) + set(ARG_TBL_GEN_SPEED "slow") + endif() + + gen_proof(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS} ${ARG_TBL_GEN_SPEED}) add_dependencies(prove_rust_examples ${example_target}_prove) if(GENERATE_EVM_VERIFIER) From fd6f68ba7ed060d60fb3c194511f6d45fe3a51cb Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 29 May 2024 19:03:07 +0000 Subject: [PATCH 29/54] handle fast_tbl generation in CMakeLists --- examples/CMakeLists.txt | 27 ++++++++++++++++++----- examples/cpp/CMakeLists.txt | 44 ++++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2450883c0..58080c3af 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -33,8 +33,11 @@ function(assign_ir) add_custom_target(${target}_generate_crct COMMAND $ -b ${binary_name} + -i ${INPUTS_DIR}/${input} + ${minus_p} ${private_input_string} -c circuit_${target}.crct -j table_pieces_${target}.json + -t assignment_${target}.tbl -e ${curve_type} --generate-type circuit ${max_num_provers_flag} ${max_num_provers_amount} @@ -92,7 +95,7 @@ function(assign_ir) -i ${INPUTS_DIR}/${input} ${minus_p} ${private_input_string} -j table_pieces_${target}.json - -t assignment_${target}.tbl -e ${curve_type} --check + -t assignment_${target}.tbl -e ${curve_type} --generate-type assignment-fast ${max_num_provers_flag} ${max_num_provers_amount} ${arithmetization_flag} ${arithmetization_amount} @@ -100,7 +103,21 @@ function(assign_ir) COMMAND_EXPAND_LISTS VERBATIM) - add_custom_target(${target}_generate_tbl_no_check + add_custom_target(${target}_generate_tbl_fast_depends_on_crct + COMMAND $ + -b ${binary_name} + -i ${INPUTS_DIR}/${input} + ${minus_p} ${private_input_string} + -j table_pieces_${target}.json + -t assignment_${target}.tbl -e ${curve_type} + --generate-type assignment-fast + ${max_num_provers_flag} ${max_num_provers_amount} + ${arithmetization_flag} ${arithmetization_amount} + DEPENDS ${target} ${INPUTS_DIR}/${input} ${target}_generate_crct $ + COMMAND_EXPAND_LISTS + VERBATIM) + + add_custom_target(${target}_generate_tbl_no_check COMMAND $ -b ${binary_name} -i ${INPUTS_DIR}/${input} @@ -168,9 +185,9 @@ function(gen_single_proof target curve_type provers_amount tbl_gen_speed) set(prover_num "") endif() - set (depends_tbl ${target}_generate_tbl) + set (depends_tbl ${target}_generate_both) if (tbl_gen_speed STREQUAL "fast") - set (depends_tbl ${target}_generate_tbl_fast) + set (depends_tbl ${target}_generate_tbl_fast_depends_on_crct) endif() add_custom_target(${target}_prove${prover_num} @@ -180,7 +197,7 @@ function(gen_single_proof target curve_type provers_amount tbl_gen_speed) -o transpiler_output_${target}${prover_num} -e ${curve_type} ${multi_prover_flag} - DEPENDS ${target}_generate_crct ${depends_tbl} $ + DEPENDS ${depends_tbl} $ COMMAND_EXPAND_LISTS VERBATIM) endfunction() diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 099188f65..c2e588000 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -1,5 +1,6 @@ add_custom_target(compile_cpp_examples) add_custom_target(cpp_examples_generate_tbl) +add_custom_target(cpp_examples_generate_tbl_fast) add_custom_target(cpp_examples_generate_tbl_no_check) add_custom_target(cpp_examples_generate_crct) add_custom_target(cpp_examples_generate_both) @@ -101,6 +102,11 @@ function(add_example_with_proving example_target) gen_proof(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS} ${ARG_TBL_GEN_SPEED}) add_dependencies(prove_cpp_examples ${example_target}_prove) + if (ARG_TBL_GEN_SPEED STREQUAL "fast") + add_dependencies(cpp_examples_generate_tbl_fast ${example_target}_generate_tbl_fast) + endif() + + if(GENERATE_EVM_VERIFIER) gen_evm_verifier(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS}) add_dependencies(prove_cpp_examples ${example_target}_evm_verifier) @@ -138,33 +144,35 @@ endfunction() add_custom_target("recursion") add_example_with_proving(arithmetics_cpp_example SOURCES arithmetics.cpp INPUT arithmetics.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_with_proving(integer_arithmetics_cpp_example SOURCES integer_arithmetics.cpp INPUT integer_arithmetics.inp CURVE_TYPE pallas) -add_example_with_proving(poseidon_cpp_example SOURCES poseidon.cpp INPUT poseidon.inp CURVE_TYPE pallas) -add_example_with_proving_and_recursion(merkle_tree_poseidon_cpp_example SOURCES merkle_tree_poseidon.cpp INPUT merkle_tree_poseidon.inp CURVE_TYPE pallas) -add_example_with_proving(merkle_poseidon_1prover_cpp_example SOURCES merkle_poseidon_1prover.cpp INPUT merkle_poseidon_1prover_public.inp PRIVATE_INPUT merkle_poseidon_1prover_private.inp CURVE_TYPE pallas) +add_example_with_proving(integer_arithmetics_cpp_example SOURCES integer_arithmetics.cpp INPUT integer_arithmetics.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(poseidon_cpp_example SOURCES poseidon.cpp INPUT poseidon.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving_and_recursion(merkle_tree_poseidon_cpp_example SOURCES merkle_tree_poseidon.cpp INPUT merkle_tree_poseidon.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(merkle_poseidon_1prover_cpp_example SOURCES merkle_poseidon_1prover.cpp INPUT merkle_poseidon_1prover_public.inp PRIVATE_INPUT merkle_poseidon_1prover_private.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(merkle_poseidon_2provers_cpp_example SOURCES merkle_poseidon_2provers.cpp INPUT merkle_poseidon_2provers_public.inp PRIVATE_INPUT merkle_poseidon_2provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 2) -add_example_with_proving_and_recursion(merkle_poseidon_3provers_cpp_example SOURCES merkle_poseidon_3provers.cpp INPUT merkle_poseidon_3provers_public.inp PRIVATE_INPUT merkle_poseidon_3provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 3) -add_example_with_proving(merkle_poseidon_5provers_cpp_example SOURCES merkle_poseidon_5provers.cpp INPUT merkle_poseidon_5provers_public.inp PRIVATE_INPUT merkle_poseidon_5provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 5) -add_example_with_proving(merkle_poseidon_21prover_3layers_cpp_example SOURCES merkle_poseidon_21prover_3layers.cpp INPUT merkle_poseidon_21prover_3layers_public.inp PRIVATE_INPUT merkle_poseidon_21prover_3layers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 21) +add_example_with_proving_and_recursion(merkle_poseidon_3provers_cpp_example SOURCES merkle_poseidon_3provers.cpp INPUT merkle_poseidon_3provers_public.inp PRIVATE_INPUT merkle_poseidon_3provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 3 TBL_GEN_SPEED fast) +add_example_with_proving(merkle_poseidon_5provers_cpp_example SOURCES merkle_poseidon_5provers.cpp INPUT merkle_poseidon_5provers_public.inp PRIVATE_INPUT merkle_poseidon_5provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 5 TBL_GEN_SPEED fast) +add_example_with_proving(merkle_poseidon_21prover_3layers_cpp_example SOURCES merkle_poseidon_21prover_3layers.cpp INPUT merkle_poseidon_21prover_3layers_public.inp PRIVATE_INPUT merkle_poseidon_21prover_3layers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 21 TBL_GEN_SPEED fast) add_example_without_proving(merkle_tree_sha2_256_cpp_example SOURCES merkle_tree_sha2_256.cpp INPUT merkle_tree_sha2_256.inp CURVE_TYPE pallas) add_example_without_proving(validate_merkle_path_sha2_256_cpp_example SOURCES validate_merkle_path_sha2_256.cpp INPUT validate_merkle_path_sha2_256.inp CURVE_TYPE pallas) add_example_without_proving(sha256_cpp_example SOURCES sha2_256.cpp INPUT sha2_256.inp CURVE_TYPE pallas) -add_example_with_proving(memory_cpp_example SOURCES memory.cpp INPUT memory.inp CURVE_TYPE pallas) -add_example_with_proving(polynomial_cpp_example SOURCES polynomial.cpp INPUT polynomial.inp COMPILER_OPTIONS -std=c++20 CURVE_TYPE pallas) +add_example_with_proving(memory_cpp_example SOURCES memory.cpp INPUT memory.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(polynomial_cpp_example SOURCES polynomial.cpp INPUT polynomial.inp COMPILER_OPTIONS -std=c++20 CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(pallas_curve_addition_cpp_example SOURCES pallas_curve_examples/pallas_curve_add.cpp INPUT pallas_curve_examples/pallas_curve_add.inp - CURVE_TYPE pallas) + CURVE_TYPE pallas + TBL_GEN_SPEED fast) add_example_with_proving(pallas_curve_multiplication_cpp_example SOURCES pallas_curve_examples/pallas_curve_mul.cpp INPUT pallas_curve_examples/pallas_curve_mul.inp - CURVE_TYPE pallas) -add_example_with_proving(ed25519_curve_add_cpp SOURCES ed25519_curve_add.cpp INPUT ed25519_curve_add.inp CURVE_TYPE pallas) + CURVE_TYPE pallas + TBL_GEN_SPEED fast) +add_example_with_proving(ed25519_curve_add_cpp SOURCES ed25519_curve_add.cpp INPUT ed25519_curve_add.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(ed25519_curve_mul_cpp SOURCES ed25519_curve_mul.cpp INPUT ed25519_curve_mul.inp CURVE_TYPE pallas) -add_example_with_proving(ed25519_field_add_cpp SOURCES ed25519_field_add.cpp INPUT ed25519_field_add.inp CURVE_TYPE pallas) +add_example_with_proving(ed25519_field_add_cpp SOURCES ed25519_field_add.cpp INPUT ed25519_field_add.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) #add_example_with_proving(strlen_cpp_example SOURCES strlen.cpp INPUT strlen.inp CURVE_TYPE pallas) -add_example_with_proving(sha2_512_cpp SOURCES sha2_512.cpp INPUT sha2_512.inp CURVE_TYPE pallas) +add_example_with_proving(sha2_512_cpp SOURCES sha2_512.cpp INPUT sha2_512.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(uint_remainder_cpp SOURCES bit_operations/uint_remainder.cpp INPUT bit_operations/uint_remainder.inp CURVE_TYPE pallas) add_example_with_proving(uint_division_cpp SOURCES bit_operations/uint_division.cpp INPUT bit_operations/uint_division.inp CURVE_TYPE pallas) @@ -173,8 +181,8 @@ add_example_with_proving(uint_shift_right SOURCES bit_operations/shift_right.cpp add_example_with_proving(uint_bit_decomposition SOURCES bit_operations/bit_decomposition.cpp INPUT bit_operations/bit_decomposition.inp CURVE_TYPE pallas) add_example_with_proving(uint_bit_composition SOURCES bit_operations/bit_composition.cpp INPUT bit_operations/bit_composition.inp CURVE_TYPE pallas) -add_example_with_proving(compare_neq_cpp SOURCES compare/neq.cpp INPUT compare/neq.inp CURVE_TYPE pallas) -add_example_with_proving(compare_eq_cpp SOURCES compare/eq.cpp INPUT compare/eq.inp CURVE_TYPE pallas) +add_example_with_proving(compare_neq_cpp SOURCES compare/neq.cpp INPUT compare/neq.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(compare_eq_cpp SOURCES compare/eq.cpp INPUT compare/eq.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(compare_grequal_cpp SOURCES compare/grequal.cpp INPUT compare/grequal.inp CURVE_TYPE pallas) add_example_with_proving(compare_lequal_cpp SOURCES compare/lequal.cpp INPUT compare/lequal.inp CURVE_TYPE pallas) add_example_with_proving(compare_greater_than_cpp SOURCES compare/greater_than.cpp INPUT compare/greater_than.inp CURVE_TYPE pallas) @@ -186,8 +194,8 @@ add_example_without_proving(balances_tree_cpp_example SOURCES balances_tree.cpp add_example_without_proving(eddsa_signature_verification_cpp SOURCES eddsa_signature_verification.cpp INPUT eddsa_signature_verification.inp CURVE_TYPE pallas) add_example_without_proving(zkbridge_cpp SOURCES zkbridge.cpp INPUT zkbridge.inp CURVE_TYPE pallas ARITHMETIZARION 15 1 5 30 50 6) -add_example_with_proving(private_input_cpp SOURCES private_input.cpp INPUT private_input_public.inp PRIVATE_INPUT private_input_private.inp CURVE_TYPE pallas) -add_example_with_proving(private_input_array_cpp SOURCES private_input_array.cpp INPUT private_input_array_public.inp PRIVATE_INPUT private_input_array_private.inp CURVE_TYPE pallas) +add_example_with_proving(private_input_cpp SOURCES private_input.cpp INPUT private_input_public.inp PRIVATE_INPUT private_input_private.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(private_input_array_cpp SOURCES private_input_array.cpp INPUT private_input_array_public.inp PRIVATE_INPUT private_input_array_private.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(fri_lin_inter SOURCES placeholder_prover/fri_lin_inter.cpp INPUT placeholder_prover/fri_lin_inter.inp CURVE_TYPE pallas) add_example_with_proving(fri_array_swap SOURCES placeholder_prover/fri_array_swap.cpp INPUT placeholder_prover/fri_array_swap.inp CURVE_TYPE pallas) From 9a4a2a6f7222c9d390b7fcf49911a64fc0a92e29 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 29 May 2024 19:04:28 +0000 Subject: [PATCH 30/54] update assigner --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 0b85d5f27..5e66cc6e6 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 0b85d5f27eae691131227c2160ec7728cd0826c2 +Subproject commit 5e66cc6e6a23dbbaec378b3310f5a5e66d20ef99 From 2e8aedf0c6c77eda3480b00bef1b1d0c64ce5533 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 29 May 2024 19:06:11 +0000 Subject: [PATCH 31/54] update blueprint --- libs/blueprint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/blueprint b/libs/blueprint index b822e1697..f3aa64608 160000 --- a/libs/blueprint +++ b/libs/blueprint @@ -1 +1 @@ -Subproject commit b822e1697a3713efb95646e428a4a26268be7922 +Subproject commit f3aa6460873527c9ae66ef60ccdf4b704f9f641d From c0dbbdbadf3dd3ede5a1187407d307d5eb1c81ff Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 29 May 2024 19:07:34 +0000 Subject: [PATCH 32/54] handle fast table printing in assigner main --- bin/assigner/src/main.cpp | 264 +++++++++++++++++++++++++------------- 1 file changed, 172 insertions(+), 92 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index a92ee890b..be9caebc9 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -400,21 +400,18 @@ std::string add_filename_prefix( return new_path.string(); } + template -void print_assignment_table_fast( +std::uint32_t padded_rows_from_table_header( const assignment_proxy &table_proxy, print_table_kind print_kind, std::string assignment_table_file_name, std::uint32_t idx ) { - if (print_kind == print_table_kind::MULTI_PROVER) { assignment_table_file_name = assignment_table_file_name + std::to_string(idx); } - std::ifstream itable_header; - std::ofstream otable_witness; - std::ofstream otable_pub_inp; itable_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); if (!itable_header) { throw std::runtime_error("Something wrong with input file header_" + assignment_table_file_name); // TODO: add catch @@ -435,56 +432,46 @@ void print_assignment_table_fast( ASSERT(status == nil::marshalling::status_type::success); std::uint32_t padded_rows_amount = std::get<5>(marshalled_table_header.value()).value(); + return padded_rows_amount; - otable_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); - if (!otable_witness) { - throw std::runtime_error("Something wrong with output " + add_filename_prefix("witness_", assignment_table_file_name)); - } - otable_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); - if (!otable_pub_inp) { - throw std::runtime_error("Something wrong with output " + add_filename_prefix("pub_inp_", assignment_table_file_name)); - } - - print_witness(table_proxy, print_kind, padded_rows_amount, otable_witness); - print_pub_inp(table_proxy, print_kind, padded_rows_amount, otable_pub_inp); } template -void print_assignment_table(const assignment_proxy &table_proxy, - print_table_kind print_kind, - std::uint32_t ComponentConstantColumns, std::uint32_t ComponentSelectorColumns, - std::string assignment_table_file_name, - std::uint32_t idx - ) { +void print_assignment_table_dynamic( + std::uint32_t padded_rows_amount, + const assignment_proxy &table_proxy, + print_table_kind print_kind, + std::string assignment_table_file_name, + std::uint32_t idx +) { if (print_kind == print_table_kind::MULTI_PROVER) { assignment_table_file_name = assignment_table_file_name + std::to_string(idx); } - std::ofstream otable_header; std::ofstream otable_witness; std::ofstream otable_pub_inp; - std::ofstream otable_constants; - std::ofstream otable_selectors; - otable_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); - if (!otable_header) - throw std::runtime_error("Something wrong with output " + add_filename_prefix("header_", assignment_table_file_name)); otable_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); - if (!otable_witness) + if (!otable_witness) { throw std::runtime_error("Something wrong with output " + add_filename_prefix("witness_", assignment_table_file_name)); + } otable_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); - if (!otable_pub_inp) + if (!otable_pub_inp) { throw std::runtime_error("Something wrong with output " + add_filename_prefix("pub_inp_", assignment_table_file_name)); - otable_constants.open(add_filename_prefix("constants_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); - if (!otable_constants) - throw std::runtime_error("Something wrong with output " + add_filename_prefix("constants_", assignment_table_file_name)); - otable_selectors.open(add_filename_prefix("selectors_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); - if (!otable_selectors) - throw std::runtime_error("Something wrong with output " + add_filename_prefix("selectors_", assignment_table_file_name)); + } + print_witness(table_proxy, print_kind, padded_rows_amount, otable_witness); + print_pub_inp(table_proxy, print_kind, padded_rows_amount, otable_pub_inp); +} + + +template +std::array extract_table_parameters_from_table( + const assignment_proxy &table_proxy, + print_table_kind print_kind +) { - using AssignmentTableType = assignment_proxy; std::uint32_t usable_rows_amount; std::uint32_t total_columns; std::uint32_t total_size; @@ -541,6 +528,59 @@ void print_assignment_table(const assignment_proxy &table_p padded_rows_amount = 8; } + std::array result = { + witness_size, + public_input_size, + shared_size, + constant_size, + selector_size, + usable_rows_amount, + padded_rows_amount + }; + + return result; +} + + +template +void print_assignment_table_static( + std::array asdf, + const assignment_proxy &table_proxy, + print_table_kind print_kind, + std::uint32_t ComponentConstantColumns, std::uint32_t ComponentSelectorColumns, + std::string assignment_table_file_name, + std::uint32_t idx + ) { + + if (print_kind == print_table_kind::MULTI_PROVER) { + assignment_table_file_name = assignment_table_file_name + std::to_string(idx); + } + + std::ofstream otable_header; + std::ofstream otable_constants; + std::ofstream otable_selectors; + + otable_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); + if (!otable_header) + throw std::runtime_error("Something wrong with output " + add_filename_prefix("header_", assignment_table_file_name)); + otable_constants.open(add_filename_prefix("constants_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); + if (!otable_constants) + throw std::runtime_error("Something wrong with output " + add_filename_prefix("constants_", assignment_table_file_name)); + otable_selectors.open(add_filename_prefix("selectors_", assignment_table_file_name), std::ios_base::binary | std::ios_base::out); + if (!otable_selectors) + throw std::runtime_error("Something wrong with output " + add_filename_prefix("selectors_", assignment_table_file_name)); + + + using AssignmentTableType = assignment_proxy; + + std::uint32_t witness_size = asdf[0]; + std::uint32_t public_input_size = asdf[1]; + std::uint32_t shared_size = asdf[2]; + std::uint32_t constant_size = asdf[3]; + std::uint32_t selector_size = asdf[4]; + std::uint32_t usable_rows_amount = asdf[5]; + std::uint32_t padded_rows_amount = asdf[6]; + print_size_t(witness_size, otable_header); print_size_t(public_input_size + shared_size, otable_header); print_size_t(constant_size, otable_header); @@ -548,16 +588,10 @@ void print_assignment_table(const assignment_proxy &table_p print_size_t(usable_rows_amount, otable_header); print_size_t(padded_rows_amount, otable_header); - print_witness(table_proxy, print_kind, padded_rows_amount, otable_witness); - print_pub_inp(table_proxy, print_kind, padded_rows_amount, otable_pub_inp); print_constants(table_proxy, print_kind, padded_rows_amount, ComponentConstantColumns, otable_constants); print_selectors(table_proxy, print_kind, padded_rows_amount, ComponentSelectorColumns, otable_selectors); + } - /* - TODO: maybe save this check in some form - ASSERT_MSG(witness_idx + pub_inp_idx + constant_idx + selector_idx == total_size, "Printed index not equal required assignment size" ); - */ -} bool read_json( std::string input_file_name, @@ -611,11 +645,15 @@ void assignment_table_printer( const std::size_t &ComponentConstantColumns, const std::size_t &ComponentSelectorColumns ) { - print_assignment_table( + std::array asdf = extract_table_parameters_from_table( + assigner_instance.assignments[idx], + print_table_kind::MULTI_PROVER + ); + + print_assignment_table_dynamic( + asdf[6], assigner_instance.assignments[idx], print_table_kind::MULTI_PROVER, - ComponentConstantColumns, - ComponentSelectorColumns, assignment_table_file_name, idx ); @@ -641,6 +679,8 @@ int curve_dependent_main(std::string bytecode_file_name, std::array column_sizes ) { + ASSERT_MSG(!(check_validity && gen_mode.has_fast_tbl()), "validity check does not work in fast table generation mode!"); + const std::size_t ComponentConstantColumns = column_sizes[2]; const std::size_t LookupConstantColumns = column_sizes[3]; const std::size_t ComponentSelectorColumns = column_sizes[4]; @@ -893,36 +933,7 @@ int curve_dependent_main(std::string bytecode_file_name, file.close(); } - if (gen_mode.has_fast_tbl()) { - if (assigner_instance.assignments.size() == 1) { - auto fast_tbl_print_start = std::chrono::high_resolution_clock::now(); - print_assignment_table_fast( - assigner_instance.assignments[0], - print_table_kind::SINGLE_PROVER, - assignment_table_file_name, - 0 - ); - auto fast_tbl_print_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - fast_tbl_print_start); - BOOST_LOG_TRIVIAL(debug) << "fast_tbl_print_duration: " << fast_tbl_print_duration.count() << "ms"; - return 0; - } - else { - auto fast_tbl_print_start = std::chrono::high_resolution_clock::now(); - // print assignment table - for (std::size_t i = 0; i < assigner_instance.assignments.size(); i++) { - print_assignment_table_fast( - assigner_instance.assignments[i], - print_table_kind::MULTI_PROVER, - assignment_table_file_name, - i - ); - } - auto fast_tbl_print_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - fast_tbl_print_start); - BOOST_LOG_TRIVIAL(debug) << "fast_tbl_print_duration: " << fast_tbl_print_duration.count() << "ms"; - return 0; - } - } - + if (gen_mode.has_circuit()) { auto pack_lookup_start = std::chrono::high_resolution_clock::now(); // pack lookup tables if (assigner_instance.circuits[0].get_reserved_tables().size() > 0) { @@ -954,27 +965,51 @@ int curve_dependent_main(std::string bytecode_file_name, } auto pack_lookup_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - pack_lookup_start); BOOST_LOG_TRIVIAL(debug) << "pack_lookup_duration: " << pack_lookup_duration.count() << "ms"; + } auto overall_table_printing_start = std::chrono::high_resolution_clock::now(); constexpr std::uint32_t invalid_target_prover = std::numeric_limits::max(); // print assignment tables and circuits - ASSERT_MSG(assigner_instance.assignments.size() == assigner_instance.circuits.size(), "Missmatch assignments circuits size"); + ASSERT_MSG(gen_mode.has_fast_tbl() || (assigner_instance.assignments.size() == assigner_instance.circuits.size()), "Missmatch assignments circuits size"); if (assigner_instance.assignments.size() == 1 && (target_prover == 0 || target_prover == invalid_target_prover)) { auto single_table_print_start = std::chrono::high_resolution_clock::now(); // print assignment table - if (gen_mode.has_assignments()) { + if (gen_mode.has_slow_tbl()) { - print_assignment_table( + std::array asdf = extract_table_parameters_from_table( + assigner_instance.assignments[0], + print_table_kind::SINGLE_PROVER + ); + + print_assignment_table_dynamic( + asdf[6], assigner_instance.assignments[0], print_table_kind::SINGLE_PROVER, - ComponentConstantColumns, - ComponentSelectorColumns, assignment_table_file_name, 0 ); auto single_table_print_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - single_table_print_start); BOOST_LOG_TRIVIAL(debug) << "single_table_print_duration: " << single_table_print_duration.count() << "ms"; } + if (gen_mode.has_fast_tbl()) { + auto fast_tbl_print_start = std::chrono::high_resolution_clock::now(); + + std::uint32_t padded_rows_amount = padded_rows_from_table_header( + assigner_instance.assignments[0], + print_table_kind::SINGLE_PROVER, + assignment_table_file_name, + 0); + + print_assignment_table_dynamic( + padded_rows_amount, + assigner_instance.assignments[0], + print_table_kind::SINGLE_PROVER, + assignment_table_file_name, + 0); + + auto fast_tbl_print_duration = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - fast_tbl_print_start); + BOOST_LOG_TRIVIAL(debug) << "fast_tbl_print_duration: " << fast_tbl_print_duration.count() << "ms"; + } // print circuit if (gen_mode.has_circuit()) { @@ -988,6 +1023,24 @@ int curve_dependent_main(std::string bytecode_file_name, print_circuit( assigner_instance.circuits[0], assigner_instance.assignments[0], false, 0, ocircuit); ocircuit.close(); + + std::array asdf = extract_table_parameters_from_table( + assigner_instance.assignments[0], + print_table_kind::SINGLE_PROVER + ); + + print_assignment_table_static( + asdf, + assigner_instance.assignments[0], + print_table_kind::SINGLE_PROVER, + ComponentConstantColumns, + ComponentSelectorColumns, + assignment_table_file_name, + 0 + ); + + + } } else if (assigner_instance.assignments.size() > 1 && (target_prover < assigner_instance.assignments.size() || target_prover == invalid_target_prover)) { @@ -998,13 +1051,7 @@ int curve_dependent_main(std::string bytecode_file_name, for (std::uint32_t idx = start_idx; idx < end_idx; idx++) { // print assignment table - if (gen_mode.has_assignments()) { - - // print_table_human_readable( - // assigner_instance.assignments[idx], - // ComponentConstantColumns, - // ComponentSelectorColumns - // ); + if (gen_mode.has_slow_tbl()) { auto future = std::async( std::launch::async, @@ -1018,7 +1065,24 @@ int curve_dependent_main(std::string bytecode_file_name, futures.push_back(std::move(future)); } + if (gen_mode.has_fast_tbl()) { + std::uint32_t padded_rows_amount = padded_rows_from_table_header( + assigner_instance.assignments[idx], + print_table_kind::MULTI_PROVER, + assignment_table_file_name, + idx); + + print_assignment_table_dynamic( + padded_rows_amount, + assigner_instance.assignments[idx], + print_table_kind::MULTI_PROVER, + assignment_table_file_name, + idx); + } + } + + for (std::uint32_t idx = start_idx; idx < end_idx; idx++) { // print circuit if (gen_mode.has_circuit()) { std::ofstream ocircuit; @@ -1033,6 +1097,21 @@ int curve_dependent_main(std::string bytecode_file_name, assigner_instance.circuits[idx], assigner_instance.assignments[idx], true, idx, ocircuit); ocircuit.close(); + + std::array asdf = extract_table_parameters_from_table( + assigner_instance.assignments[idx], + print_table_kind::MULTI_PROVER + ); + + print_assignment_table_static( + asdf, + assigner_instance.assignments[idx], + print_table_kind::MULTI_PROVER, + ComponentConstantColumns, + ComponentSelectorColumns, + assignment_table_file_name, + idx + ); } } @@ -1161,13 +1240,13 @@ int main(int argc, char *argv[]) { return 1; } - nil::blueprint::generation_mode gen_mode = nil::blueprint::generation_mode::assignments() | nil::blueprint::generation_mode::circuit(); + nil::blueprint::generation_mode gen_mode = nil::blueprint::generation_mode::slow_tbl() | nil::blueprint::generation_mode::circuit(); if (vm.count("generate-type")) { const auto generate_type = vm["generate-type"].as(); if (generate_type == "circuit") { gen_mode = nil::blueprint::generation_mode::circuit(); } else if (generate_type == "assignment") { - gen_mode = nil::blueprint::generation_mode::assignments(); + gen_mode = nil::blueprint::generation_mode::slow_tbl(); } else if (generate_type == "assignment-fast") { gen_mode = nil::blueprint::generation_mode::fast_tbl(); } else if (generate_type == "size_estimation") { @@ -1180,6 +1259,7 @@ int main(int argc, char *argv[]) { return 1; } } + gen_mode = gen_mode | nil::blueprint::generation_mode::assignments(); // for now in any case we generate assignment table to determine used_rows if (!vm.count("public-input") && !vm.count("private-input") && gen_mode.has_assignments()) { std::cerr << "Both public and private input file names are not specified" << std::endl; @@ -1204,7 +1284,7 @@ int main(int argc, char *argv[]) { public_input_file_name = vm["public-input"].as(); } - if (gen_mode.has_assignments() || gen_mode.has_fast_tbl()) { + if (gen_mode.has_circuit() || gen_mode.has_fast_tbl() || gen_mode.has_slow_tbl()) { if (vm.count("assignment-table")) { assignment_table_file_name = vm["assignment-table"].as(); } else { From 0c3726a0e99a5868e544f1befc66042d8ea2a674 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 29 May 2024 21:15:42 +0000 Subject: [PATCH 33/54] dont pack lookup tables in the fast tbl case --- bin/assigner/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index be9caebc9..afb0bfbb5 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -933,7 +933,7 @@ int curve_dependent_main(std::string bytecode_file_name, file.close(); } - if (gen_mode.has_circuit()) { + if (!gen_mode.has_fast_tbl()) { auto pack_lookup_start = std::chrono::high_resolution_clock::now(); // pack lookup tables if (assigner_instance.circuits[0].get_reserved_tables().size() > 0) { From 8778167526324ecde7c9d360f6693deea7bede2e Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 29 May 2024 22:19:33 +0000 Subject: [PATCH 34/54] disable size estimation step in CI --- .github/workflows/build_linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index a4566c3fd..9a37ab18d 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -183,9 +183,9 @@ jobs: run: | bash ./tests/compare_folders_content.sh build/separate_generatuon build/simultaneous_generation - - name: Run size estimation for C++ examples - run: | - cmake --build build -t cpp_examples_estimate_size + # - name: Run size estimation for C++ examples + # run: | + # cmake --build build -t cpp_examples_estimate_size - name: Copy examples' circuits and assignments for uploading run: | From 0bc863f20061f506d5b8231b4250da7d2b766329 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 5 Jun 2024 16:47:58 +0000 Subject: [PATCH 35/54] update assigner --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 5e66cc6e6..c3a8d443b 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 5e66cc6e6a23dbbaec378b3310f5a5e66d20ef99 +Subproject commit c3a8d443b2cc13ce44c1eb65abe0518d53caf56b From 969593974705321ccf28c18e0ba775178d0a5076 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 5 Jun 2024 16:48:11 +0000 Subject: [PATCH 36/54] update blueprint --- libs/blueprint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/blueprint b/libs/blueprint index f3aa64608..0d87a281a 160000 --- a/libs/blueprint +++ b/libs/blueprint @@ -1 +1 @@ -Subproject commit f3aa6460873527c9ae66ef60ccdf4b704f9f641d +Subproject commit 0d87a281a615c66a804b260764b20b62b3157afe From 3e625a885f8900c7857d9b90449b3fabac04fe3e Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Wed, 5 Jun 2024 16:49:40 +0000 Subject: [PATCH 37/54] add more fast examples --- examples/cpp/CMakeLists.txt | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index c2e588000..f94e6a291 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -148,14 +148,14 @@ add_example_with_proving(integer_arithmetics_cpp_example SOURCES integer_arithme add_example_with_proving(poseidon_cpp_example SOURCES poseidon.cpp INPUT poseidon.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving_and_recursion(merkle_tree_poseidon_cpp_example SOURCES merkle_tree_poseidon.cpp INPUT merkle_tree_poseidon.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(merkle_poseidon_1prover_cpp_example SOURCES merkle_poseidon_1prover.cpp INPUT merkle_poseidon_1prover_public.inp PRIVATE_INPUT merkle_poseidon_1prover_private.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_with_proving(merkle_poseidon_2provers_cpp_example SOURCES merkle_poseidon_2provers.cpp INPUT merkle_poseidon_2provers_public.inp PRIVATE_INPUT merkle_poseidon_2provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 2) +add_example_with_proving(merkle_poseidon_2provers_cpp_example SOURCES merkle_poseidon_2provers.cpp INPUT merkle_poseidon_2provers_public.inp PRIVATE_INPUT merkle_poseidon_2provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 2 TBL_GEN_SPEED fast) add_example_with_proving_and_recursion(merkle_poseidon_3provers_cpp_example SOURCES merkle_poseidon_3provers.cpp INPUT merkle_poseidon_3provers_public.inp PRIVATE_INPUT merkle_poseidon_3provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 3 TBL_GEN_SPEED fast) add_example_with_proving(merkle_poseidon_5provers_cpp_example SOURCES merkle_poseidon_5provers.cpp INPUT merkle_poseidon_5provers_public.inp PRIVATE_INPUT merkle_poseidon_5provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 5 TBL_GEN_SPEED fast) add_example_with_proving(merkle_poseidon_21prover_3layers_cpp_example SOURCES merkle_poseidon_21prover_3layers.cpp INPUT merkle_poseidon_21prover_3layers_public.inp PRIVATE_INPUT merkle_poseidon_21prover_3layers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 21 TBL_GEN_SPEED fast) -add_example_without_proving(merkle_tree_sha2_256_cpp_example SOURCES merkle_tree_sha2_256.cpp INPUT merkle_tree_sha2_256.inp CURVE_TYPE pallas) -add_example_without_proving(validate_merkle_path_sha2_256_cpp_example SOURCES validate_merkle_path_sha2_256.cpp INPUT validate_merkle_path_sha2_256.inp CURVE_TYPE pallas) -add_example_without_proving(sha256_cpp_example SOURCES sha2_256.cpp INPUT sha2_256.inp CURVE_TYPE pallas) +add_example_without_proving(merkle_tree_sha2_256_cpp_example SOURCES merkle_tree_sha2_256.cpp INPUT merkle_tree_sha2_256.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_without_proving(validate_merkle_path_sha2_256_cpp_example SOURCES validate_merkle_path_sha2_256.cpp INPUT validate_merkle_path_sha2_256.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_without_proving(sha256_cpp_example SOURCES sha2_256.cpp INPUT sha2_256.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(memory_cpp_example SOURCES memory.cpp INPUT memory.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(polynomial_cpp_example SOURCES polynomial.cpp INPUT polynomial.inp COMPILER_OPTIONS -std=c++20 CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(pallas_curve_addition_cpp_example @@ -169,41 +169,41 @@ add_example_with_proving(pallas_curve_multiplication_cpp_example CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(ed25519_curve_add_cpp SOURCES ed25519_curve_add.cpp INPUT ed25519_curve_add.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_with_proving(ed25519_curve_mul_cpp SOURCES ed25519_curve_mul.cpp INPUT ed25519_curve_mul.inp CURVE_TYPE pallas) +add_example_with_proving(ed25519_curve_mul_cpp SOURCES ed25519_curve_mul.cpp INPUT ed25519_curve_mul.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(ed25519_field_add_cpp SOURCES ed25519_field_add.cpp INPUT ed25519_field_add.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -#add_example_with_proving(strlen_cpp_example SOURCES strlen.cpp INPUT strlen.inp CURVE_TYPE pallas) +add_example_with_proving(strlen_cpp_example SOURCES strlen.cpp INPUT strlen.inp CURVE_TYPE pallas) add_example_with_proving(sha2_512_cpp SOURCES sha2_512.cpp INPUT sha2_512.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_with_proving(uint_remainder_cpp SOURCES bit_operations/uint_remainder.cpp INPUT bit_operations/uint_remainder.inp CURVE_TYPE pallas) -add_example_with_proving(uint_division_cpp SOURCES bit_operations/uint_division.cpp INPUT bit_operations/uint_division.inp CURVE_TYPE pallas) -add_example_with_proving(uint_shift_left SOURCES bit_operations/shift_left.cpp INPUT bit_operations/shift_left.inp CURVE_TYPE pallas) -add_example_with_proving(uint_shift_right SOURCES bit_operations/shift_right.cpp INPUT bit_operations/shift_right.inp CURVE_TYPE pallas) -add_example_with_proving(uint_bit_decomposition SOURCES bit_operations/bit_decomposition.cpp INPUT bit_operations/bit_decomposition.inp CURVE_TYPE pallas) -add_example_with_proving(uint_bit_composition SOURCES bit_operations/bit_composition.cpp INPUT bit_operations/bit_composition.inp CURVE_TYPE pallas) +add_example_with_proving(uint_remainder_cpp SOURCES bit_operations/uint_remainder.cpp INPUT bit_operations/uint_remainder.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(uint_division_cpp SOURCES bit_operations/uint_division.cpp INPUT bit_operations/uint_division.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(uint_shift_left SOURCES bit_operations/shift_left.cpp INPUT bit_operations/shift_left.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(uint_shift_right SOURCES bit_operations/shift_right.cpp INPUT bit_operations/shift_right.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(uint_bit_decomposition SOURCES bit_operations/bit_decomposition.cpp INPUT bit_operations/bit_decomposition.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(uint_bit_composition SOURCES bit_operations/bit_composition.cpp INPUT bit_operations/bit_composition.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(compare_neq_cpp SOURCES compare/neq.cpp INPUT compare/neq.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(compare_eq_cpp SOURCES compare/eq.cpp INPUT compare/eq.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_with_proving(compare_grequal_cpp SOURCES compare/grequal.cpp INPUT compare/grequal.inp CURVE_TYPE pallas) -add_example_with_proving(compare_lequal_cpp SOURCES compare/lequal.cpp INPUT compare/lequal.inp CURVE_TYPE pallas) -add_example_with_proving(compare_greater_than_cpp SOURCES compare/greater_than.cpp INPUT compare/greater_than.inp CURVE_TYPE pallas) -add_example_with_proving(compare_less_than_cpp SOURCES compare/less_than.cpp INPUT compare/less_than.inp CURVE_TYPE pallas) +add_example_with_proving(compare_grequal_cpp SOURCES compare/grequal.cpp INPUT compare/grequal.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(compare_lequal_cpp SOURCES compare/lequal.cpp INPUT compare/lequal.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(compare_greater_than_cpp SOURCES compare/greater_than.cpp INPUT compare/greater_than.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(compare_less_than_cpp SOURCES compare/less_than.cpp INPUT compare/less_than.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) #Balances tree example is too big to be proven in a reasonable time -add_example_without_proving(balances_tree_cpp_example SOURCES balances_tree.cpp INPUT balances_tree_public.inp PRIVATE_INPUT balances_tree_private.inp CURVE_TYPE pallas) +add_example_without_proving(balances_tree_cpp_example SOURCES balances_tree.cpp INPUT balances_tree_public.inp PRIVATE_INPUT balances_tree_private.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_without_proving(eddsa_signature_verification_cpp SOURCES eddsa_signature_verification.cpp INPUT eddsa_signature_verification.inp CURVE_TYPE pallas) -add_example_without_proving(zkbridge_cpp SOURCES zkbridge.cpp INPUT zkbridge.inp CURVE_TYPE pallas ARITHMETIZARION 15 1 5 30 50 6) +add_example_without_proving(eddsa_signature_verification_cpp SOURCES eddsa_signature_verification.cpp INPUT eddsa_signature_verification.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_without_proving(zkbridge_cpp SOURCES zkbridge.cpp INPUT zkbridge.inp CURVE_TYPE pallas TBL_GEN_SPEED fast ARITHMETIZARION 15 1 5 30 50 6) add_example_with_proving(private_input_cpp SOURCES private_input.cpp INPUT private_input_public.inp PRIVATE_INPUT private_input_private.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(private_input_array_cpp SOURCES private_input_array.cpp INPUT private_input_array_public.inp PRIVATE_INPUT private_input_array_private.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_with_proving(fri_lin_inter SOURCES placeholder_prover/fri_lin_inter.cpp INPUT placeholder_prover/fri_lin_inter.inp CURVE_TYPE pallas) -add_example_with_proving(fri_array_swap SOURCES placeholder_prover/fri_array_swap.cpp INPUT placeholder_prover/fri_array_swap.inp CURVE_TYPE pallas) -add_example_with_proving(fri_array_swap_intrinsic SOURCES placeholder_prover/fri_array_swap_intrinsic.cpp INPUT placeholder_prover/fri_array_swap_intrinsic.inp CURVE_TYPE pallas) -add_example_with_proving(fri_cosets SOURCES placeholder_prover/fri_cosets.cpp INPUT placeholder_prover/fri_cosets.inp CURVE_TYPE pallas) -add_example_with_proving(gate_argument_verifier SOURCES placeholder_prover/gate_argument_verifier.cpp INPUT placeholder_prover/gate_argument_verifier.inp CURVE_TYPE pallas) -add_example_with_proving(permutation_argument_verifier SOURCES placeholder_prover/permutation_argument_verifier.cpp INPUT placeholder_prover/permutation_argument_verifier.inp CURVE_TYPE pallas) -add_example_with_proving(lookup_argument_verifier SOURCES placeholder_prover/lookup_argument_verifier.cpp INPUT placeholder_prover/lookup_argument_verifier.inp CURVE_TYPE pallas) +add_example_with_proving(fri_lin_inter SOURCES placeholder_prover/fri_lin_inter.cpp INPUT placeholder_prover/fri_lin_inter.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(fri_array_swap SOURCES placeholder_prover/fri_array_swap.cpp INPUT placeholder_prover/fri_array_swap.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(fri_array_swap_intrinsic SOURCES placeholder_prover/fri_array_swap_intrinsic.cpp INPUT placeholder_prover/fri_array_swap_intrinsic.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(fri_cosets SOURCES placeholder_prover/fri_cosets.cpp INPUT placeholder_prover/fri_cosets.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(gate_argument_verifier SOURCES placeholder_prover/gate_argument_verifier.cpp INPUT placeholder_prover/gate_argument_verifier.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(permutation_argument_verifier SOURCES placeholder_prover/permutation_argument_verifier.cpp INPUT placeholder_prover/permutation_argument_verifier.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) +add_example_with_proving(lookup_argument_verifier SOURCES placeholder_prover/lookup_argument_verifier.cpp INPUT placeholder_prover/lookup_argument_verifier.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) # add_example_without_proving(bls12_381_field_add_cpp SOURCES bls12_381_field_add.cpp INPUT bls12_381_field_add.inp CURVE_TYPE bls12381) # add_example_without_proving(bls12_381_field_mul_cpp SOURCES bls12_381_field_mul.cpp INPUT bls12_381_field_mul.inp CURVE_TYPE bls12381) From be8544d39054105c0d6386ddb850dd8647920805 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 7 Jun 2024 11:08:10 +0000 Subject: [PATCH 38/54] disable mistakenly enabled example --- examples/cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index f94e6a291..881f2dc8b 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -171,7 +171,7 @@ add_example_with_proving(pallas_curve_multiplication_cpp_example add_example_with_proving(ed25519_curve_add_cpp SOURCES ed25519_curve_add.cpp INPUT ed25519_curve_add.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(ed25519_curve_mul_cpp SOURCES ed25519_curve_mul.cpp INPUT ed25519_curve_mul.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(ed25519_field_add_cpp SOURCES ed25519_field_add.cpp INPUT ed25519_field_add.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -add_example_with_proving(strlen_cpp_example SOURCES strlen.cpp INPUT strlen.inp CURVE_TYPE pallas) +# add_example_with_proving(strlen_cpp_example SOURCES strlen.cpp INPUT strlen.inp CURVE_TYPE pallas) add_example_with_proving(sha2_512_cpp SOURCES sha2_512.cpp INPUT sha2_512.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(uint_remainder_cpp SOURCES bit_operations/uint_remainder.cpp INPUT bit_operations/uint_remainder.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) From 2f202f51e43cd4806c6ced8c8ba666f12d5823cf Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Fri, 7 Jun 2024 16:33:33 +0000 Subject: [PATCH 39/54] update blueprint --- libs/blueprint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/blueprint b/libs/blueprint index 0d87a281a..9e9bccd09 160000 --- a/libs/blueprint +++ b/libs/blueprint @@ -1 +1 @@ -Subproject commit 0d87a281a615c66a804b260764b20b62b3157afe +Subproject commit 9e9bccd09ed1e0da26d74515f8eaae6658c9b56e From 0f71241aa2e49bfb9ef7424c0245d85973107724 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 11:10:49 +0000 Subject: [PATCH 40/54] update proof producer version in CI --- .github/workflows/build_linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index 9a37ab18d..36274f9d4 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -327,11 +327,11 @@ jobs: always() && !cancelled() && (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') && needs.build-and-test-linux.result == 'success' - uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@ef8cd9152b4bec871e7efdc1d6b606e445bad274 + uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@f450e9219b0b77fa32ce27c5e9a9d9144cc577dd with: artifact-name: ${{ needs.build-and-test-linux.outputs.examples-artifact-name }} # Update next line if you need new version of proof producer - proof-producer-ref: ef8cd9152b4bec871e7efdc1d6b606e445bad274 + proof-producer-ref: f450e9219b0b77fa32ce27c5e9a9d9144cc577dd refs: ${{ needs.handle-syncwith.outputs.prs-refs }} targets: ${{ needs.build-and-test-linux.outputs.prover-targets }} From db6b4200939ca059f53133cfffc222998f68693e Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 11:11:11 +0000 Subject: [PATCH 41/54] update assigner --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index c3a8d443b..43f59869d 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit c3a8d443b2cc13ce44c1eb65abe0518d53caf56b +Subproject commit 43f59869d85fcbdd6e49d88afba9e48e74e9c308 From 435e13d011bb0e14956507c67f904582633b46ec Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 11:13:19 +0000 Subject: [PATCH 42/54] update blueprint --- libs/blueprint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/blueprint b/libs/blueprint index 9e9bccd09..52e62046a 160000 --- a/libs/blueprint +++ b/libs/blueprint @@ -1 +1 @@ -Subproject commit 9e9bccd09ed1e0da26d74515f8eaae6658c9b56e +Subproject commit 52e62046a49775651fe12d3c823234aa5949abca From d2395a86bfefa4f242ee4f867e9ea0fa41caaf89 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 11:14:22 +0000 Subject: [PATCH 43/54] update table reding in transpiler --- bin/transpiler/src/main.cpp | 92 ++++++++++++------------------------- 1 file changed, 30 insertions(+), 62 deletions(-) diff --git a/bin/transpiler/src/main.cpp b/bin/transpiler/src/main.cpp index 65a0e6258..b78ecc542 100644 --- a/bin/transpiler/src/main.cpp +++ b/bin/transpiler/src/main.cpp @@ -256,6 +256,29 @@ std::string add_filename_prefix( return new_path.string(); } +bool read_file_to_vector ( + std::vector& result_vector, + const std::string& prefix, + const std::string& assignment_table_file_name + +) { + std::cout << "assignment_table_file_name: " << assignment_table_file_name << "\n"; + std::ifstream icolumn; + icolumn.open(add_filename_prefix(prefix, assignment_table_file_name), std::ios_base::binary | std::ios_base::in); + if (!icolumn) { + std::cout << "Cannot open " << add_filename_prefix("header_", assignment_table_file_name) << std::endl; + return false; + } + icolumn.seekg(0, std::ios_base::end); + const auto input_size = icolumn.tellg(); + std::size_t old_size = result_vector.size(); + result_vector.resize(old_size + input_size); + icolumn.seekg(0, std::ios_base::beg); + icolumn.read(reinterpret_cast(result_vector.data() + old_size), input_size); + icolumn.close(); + return true; +} + template int curve_dependent_main( boost::program_options::options_description options_desc, @@ -375,69 +398,14 @@ int curve_dependent_main( AssignmentTableType assignment_table; { - std::ifstream iassignment_header; - std::ifstream iassignment_witness; - std::ifstream iassignment_pub_inp; - std::ifstream iassignment_constants; - std::ifstream iassignment_selectors; - - iassignment_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); - if (!iassignment_header) { - std::cout << "Cannot open " << add_filename_prefix("header_", assignment_table_file_name) << std::endl; - return 1; - } - iassignment_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); - if (!iassignment_witness) { - std::cout << "Cannot open " << add_filename_prefix("witness_", assignment_table_file_name) << std::endl; - return 1; - } - iassignment_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); - if (!iassignment_pub_inp) { - std::cout << "Cannot open " << add_filename_prefix("pub_inp_", assignment_table_file_name) << std::endl; - return 1; - } - iassignment_constants.open(add_filename_prefix("constants_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); - if (!iassignment_constants) { - std::cout << "Cannot open " << add_filename_prefix("constants_", assignment_table_file_name) << std::endl; - return 1; - } - iassignment_selectors.open(add_filename_prefix("selectors_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in); - if (!iassignment_selectors) { - std::cout << "Cannot open " << add_filename_prefix("selectors_", assignment_table_file_name) << std::endl; - return 1; - } + std::vector v; - iassignment_header.seekg(0, std::ios_base::end); - iassignment_witness.seekg(0, std::ios_base::end); - iassignment_pub_inp.seekg(0, std::ios_base::end); - iassignment_constants.seekg(0, std::ios_base::end); - iassignment_selectors.seekg(0, std::ios_base::end); - - const auto header_size = iassignment_header.tellg(); - const auto w_size = iassignment_witness.tellg(); - const auto pi_size = iassignment_pub_inp.tellg(); - const auto c_size = iassignment_constants.tellg(); - const auto s_size = iassignment_selectors.tellg(); - - v.resize(header_size + w_size + pi_size + c_size + s_size); - - iassignment_header.seekg(0, std::ios_base::beg); - iassignment_witness.seekg(0, std::ios_base::beg); - iassignment_pub_inp.seekg(0, std::ios_base::beg); - iassignment_constants.seekg(0, std::ios_base::beg); - iassignment_selectors.seekg(0, std::ios_base::beg); - - iassignment_header.read(reinterpret_cast(v.data()), header_size); - iassignment_witness.read(reinterpret_cast(v.data()) + header_size, w_size); - iassignment_pub_inp.read(reinterpret_cast(v.data()) + header_size + w_size, pi_size); - iassignment_constants.read(reinterpret_cast(v.data()) + header_size + w_size + pi_size, c_size); - iassignment_selectors.read(reinterpret_cast(v.data()) + header_size + w_size + pi_size + c_size, s_size); - - iassignment_header.close(); - iassignment_witness.close(); - iassignment_pub_inp.close(); - iassignment_constants.close(); - iassignment_selectors.close(); + + ASSERT(read_file_to_vector(v, "header_", assignment_table_file_name)); + ASSERT(read_file_to_vector(v, "witness_", assignment_table_file_name)); + ASSERT(read_file_to_vector(v, "pub_inp_", assignment_table_file_name)); + ASSERT(read_file_to_vector(v, "constants_", assignment_table_file_name)); + ASSERT(read_file_to_vector(v, "selectors_", assignment_table_file_name)); table_value_marshalling_type marshalled_table_data; auto read_iter = v.begin(); From d87d114d88ebbf7a34946b9883b5d6af9c984331 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 11:23:16 +0000 Subject: [PATCH 44/54] update assigner(try to fix mac CI) --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 43f59869d..6b464078f 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 43f59869d85fcbdd6e49d88afba9e48e74e9c308 +Subproject commit 6b464078f148f4780d728a00517e0a931d0451ab From dd259d56b30e1b18c9b2b07fb16cc717599a33be Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 13:59:41 +0000 Subject: [PATCH 45/54] move fast tbl stuff into assigner --- bin/assigner/src/main.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index afb0bfbb5..7a4aeef7f 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -894,14 +894,13 @@ int curve_dependent_main(std::string bytecode_file_name, return 0; } - if (gen_mode.has_circuit()) { // if we are generation circuit then we are generation table pieces in the same time. Need to write itno file + if (gen_mode.has_circuit()) { boost::json::object top_level_json; top_level_json["provers_amount"] = assigner_instance.assignments.size(); boost::json::array pieces_json; - // for (const auto& piece : nil::blueprint::table_pieces) { - for (const auto& piece : nil::blueprint::table_pieces) { + for (const auto& piece : assigner_instance.get_table_pieces()) { pieces_json.push_back(piece.to_json()); } top_level_json["table_pieces"] = pieces_json; @@ -918,7 +917,7 @@ int curve_dependent_main(std::string bytecode_file_name, boost::json::array to_be_shared_json; - for (const auto& pair : nil::blueprint::to_be_shared) { + for (const auto& pair : assigner_instance.get_to_be_shared()) { boost::json::object pair_json; pair_json["first"] = pair.first; pair_json["second"] = pair.second.to_json(); From 0f299e55447926e05ec8870eb8af2c7bfb3fcad2 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 14:00:21 +0000 Subject: [PATCH 46/54] update assigner(move fast tbl stuff into assigner) --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 6b464078f..5821a3a1c 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 6b464078f148f4780d728a00517e0a931d0451ab +Subproject commit 5821a3a1c88172d7bf797f615589eafbd60b5c80 From 76791c838c49ebaa8f1805b29c0dff46a76dd56d Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 14:24:55 +0000 Subject: [PATCH 47/54] update assigner(fix size estimation mode) --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 5821a3a1c..178acdd6b 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 5821a3a1c88172d7bf797f615589eafbd60b5c80 +Subproject commit 178acdd6b79a04e059545af4afd3b0d099c3fad6 From 389f25d31ad96fdc0d270c8993dea77255d674be Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 14:25:21 +0000 Subject: [PATCH 48/54] fix size estimation mode --- bin/assigner/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 7a4aeef7f..2c4e1c051 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -1260,7 +1260,7 @@ int main(int argc, char *argv[]) { } gen_mode = gen_mode | nil::blueprint::generation_mode::assignments(); // for now in any case we generate assignment table to determine used_rows - if (!vm.count("public-input") && !vm.count("private-input") && gen_mode.has_assignments()) { + if (!vm.count("public-input") && !vm.count("private-input") && !gen_mode.has_size_estimation()) { std::cerr << "Both public and private input file names are not specified" << std::endl; std::cout << options_desc << std::endl; return 1; From 7e2176fb05b4fa90f7bd8576308862de296fc446 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 16:05:56 +0000 Subject: [PATCH 49/54] update assigner (handle fast tbl generation with native bls field) --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index 178acdd6b..cb5cdea4b 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit 178acdd6b79a04e059545af4afd3b0d099c3fad6 +Subproject commit cb5cdea4bd893677fd81b5129a088f017b3d884a From 75958cfb46119f0e0f7eee481fde6faecd77362a Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Tue, 11 Jun 2024 16:06:41 +0000 Subject: [PATCH 50/54] enable bls examples --- bin/assigner/src/main.cpp | 38 ++++++++++++++++++------------------- examples/cpp/CMakeLists.txt | 10 +++++----- tests/cpp/CMakeLists.txt | 4 ++-- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index 2c4e1c051..ebe0b4e3f 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -1477,25 +1477,25 @@ int main(int argc, char *argv[]) { break; } case 3: { - UNREACHABLE("bls12381 curve based circuits are not supported yet"); - // return curve_dependent_main>( - // bytecode_file_name, - // public_input_file_name, - // private_input_file_name, - // assignment_table_file_name, - // circuit_file_name, - // processed_public_input_file_name, - // stack_size, - // vm.count("check"), - // log_options[log_level], - // policy, - // gen_mode, - // max_num_provers, - // max_lookup_rows, - // target_prover, - // circuit_output_print_format, - // column_sizes - // ); + return curve_dependent_main>( + bytecode_file_name, + public_input_file_name, + private_input_file_name, + assignment_table_file_name, + circuit_file_name, + table_pieces_file_name, + processed_public_input_file_name, + stack_size, + vm.count("check"), + log_options[log_level], + policy, + gen_mode, + max_num_provers, + max_lookup_rows, + target_prover, + circuit_output_print_format, + column_sizes + ); break; } }; diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 881f2dc8b..9f1dcb292 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -205,8 +205,8 @@ add_example_with_proving(gate_argument_verifier SOURCES placeholder_prover/gate_ add_example_with_proving(permutation_argument_verifier SOURCES placeholder_prover/permutation_argument_verifier.cpp INPUT placeholder_prover/permutation_argument_verifier.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) add_example_with_proving(lookup_argument_verifier SOURCES placeholder_prover/lookup_argument_verifier.cpp INPUT placeholder_prover/lookup_argument_verifier.inp CURVE_TYPE pallas TBL_GEN_SPEED fast) -# add_example_without_proving(bls12_381_field_add_cpp SOURCES bls12_381_field_add.cpp INPUT bls12_381_field_add.inp CURVE_TYPE bls12381) -# add_example_without_proving(bls12_381_field_mul_cpp SOURCES bls12_381_field_mul.cpp INPUT bls12_381_field_mul.inp CURVE_TYPE bls12381) -# add_example_without_proving(bls12_381_field_sub_cpp SOURCES bls12_381_field_sub.cpp INPUT bls12_381_field_sub.inp CURVE_TYPE bls12381) -# add_example_without_proving(bls12_381_signature_verification_cpp SOURCES bls12_381_signature_verification.cpp INPUT bls12_381_signature_verification.inp CURVE_TYPE bls12381) -# add_example_without_proving(bls12_381_signature_verification_aggregated_cpp SOURCES bls12_381_signature_verification_aggregated.cpp INPUT bls12_381_signature_verification_aggregated.inp CURVE_TYPE bls12381) +add_example_without_proving(bls12_381_field_add_cpp SOURCES bls12_381_field_add.cpp INPUT bls12_381_field_add.inp CURVE_TYPE bls12381) +add_example_without_proving(bls12_381_field_mul_cpp SOURCES bls12_381_field_mul.cpp INPUT bls12_381_field_mul.inp CURVE_TYPE bls12381) +add_example_without_proving(bls12_381_field_sub_cpp SOURCES bls12_381_field_sub.cpp INPUT bls12_381_field_sub.inp CURVE_TYPE bls12381) +add_example_without_proving(bls12_381_signature_verification_cpp SOURCES bls12_381_signature_verification.cpp INPUT bls12_381_signature_verification.inp CURVE_TYPE bls12381) +add_example_without_proving(bls12_381_signature_verification_aggregated_cpp SOURCES bls12_381_signature_verification_aggregated.cpp INPUT bls12_381_signature_verification_aggregated.inp CURVE_TYPE bls12381) diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index be34d4d76..0402b3eee 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -263,8 +263,8 @@ add_zkllvm_unit_test("algebra/recursive_prover/lookup_arg_verifier/test1/test") add_zkllvm_unit_test("algebra/recursive_prover/lookup_arg_verifier/test2/test") # bls tests -# add_zkllvm_unit_test_bls12381("algebra/fields/bls12381_fp12_arithmetics/mul") -# add_zkllvm_unit_test_bls12381("bls_signature/pairing/pairing") +add_zkllvm_unit_test_bls12381("algebra/fields/bls12381_fp12_arithmetics/mul") +add_zkllvm_unit_test_bls12381("bls_signature/pairing/pairing") add_zkllvm_unit_test("nested_if/first") add_zkllvm_unit_test("nested_if/second") From 91cf0220c0f4ad3af440a64d64eb1a1a9ffbcc8d Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 13 Jun 2024 09:33:48 +0000 Subject: [PATCH 51/54] fix size_estimation --- bin/assigner/src/main.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bin/assigner/src/main.cpp b/bin/assigner/src/main.cpp index ebe0b4e3f..b823df887 100644 --- a/bin/assigner/src/main.cpp +++ b/bin/assigner/src/main.cpp @@ -1249,7 +1249,7 @@ int main(int argc, char *argv[]) { } else if (generate_type == "assignment-fast") { gen_mode = nil::blueprint::generation_mode::fast_tbl(); } else if (generate_type == "size_estimation") { - gen_mode = nil::blueprint::generation_mode::size_estimation(); + gen_mode = nil::blueprint::generation_mode::size_estimation() | nil::blueprint::generation_mode::circuit(); } else if (generate_type == "public-input-column") { gen_mode = nil::blueprint::generation_mode::public_input_column(); } else if (generate_type != "circuit-assignment") { @@ -1287,9 +1287,11 @@ int main(int argc, char *argv[]) { if (vm.count("assignment-table")) { assignment_table_file_name = vm["assignment-table"].as(); } else { - std::cerr << "Invalid command line argument - assignment table file name is not specified" << std::endl; - std::cout << options_desc << std::endl; - return 1; + if (!gen_mode.has_size_estimation()) { + std::cerr << "Invalid command line argument - assignment table file name is not specified" << std::endl; + std::cout << options_desc << std::endl; + return 1; + } } } @@ -1297,9 +1299,11 @@ int main(int argc, char *argv[]) { if (vm.count("circuit")) { circuit_file_name = vm["circuit"].as(); } else { - std::cerr << "Invalid command line argument - circuit file name is not specified" << std::endl; - std::cout << options_desc << std::endl; - return 1; + if (!gen_mode.has_size_estimation()) { + std::cerr << "Invalid command line argument - circuit file name is not specified" << std::endl; + std::cout << options_desc << std::endl; + return 1; + } } } From 2a5a3e2dc1a3f1166eafaf7a79d8360745426f7e Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 13 Jun 2024 09:36:31 +0000 Subject: [PATCH 52/54] fix size estimation in CMakeLists --- examples/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 58080c3af..a7b3d8351 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -133,6 +133,9 @@ function(assign_ir) add_custom_target(${target}_estimate_size COMMAND $ -b ${binary_name} + -i ${INPUTS_DIR}/${input} + -j table_pieces_${target}.json + ${minus_p} ${private_input_string} -e ${curve_type} --generate-type size_estimation ${max_num_provers_flag} ${max_num_provers_amount} From 8009e31bf33d8318be04f0615716a11ace9ed5b3 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 13 Jun 2024 09:36:55 +0000 Subject: [PATCH 53/54] update assginer (size estimation fix) --- libs/assigner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/assigner b/libs/assigner index cb5cdea4b..f26adf483 160000 --- a/libs/assigner +++ b/libs/assigner @@ -1 +1 @@ -Subproject commit cb5cdea4bd893677fd81b5129a088f017b3d884a +Subproject commit f26adf483af4be46164a7176ac3ae82311697415 From c5b9bca5835f36414fe8b1c088d857f0228e6a67 Mon Sep 17 00:00:00 2001 From: Aleksandr Cherenkov Date: Thu, 13 Jun 2024 09:37:34 +0000 Subject: [PATCH 54/54] Revert "disable size estimation step in CI" This reverts commit 8778167526324ecde7c9d360f6693deea7bede2e. --- .github/workflows/build_linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index 36274f9d4..ea8897443 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -183,9 +183,9 @@ jobs: run: | bash ./tests/compare_folders_content.sh build/separate_generatuon build/simultaneous_generation - # - name: Run size estimation for C++ examples - # run: | - # cmake --build build -t cpp_examples_estimate_size + - name: Run size estimation for C++ examples + run: | + cmake --build build -t cpp_examples_estimate_size - name: Copy examples' circuits and assignments for uploading run: |