577 fast tbl

.github/workflows/build_linux.yml

@@ -327,11 +327,11 @@ jobs:
       always() && !cancelled() &&
       (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
       needs.build-and-test-linux.result == 'success'
-    uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@ef8cd9152b4bec871e7efdc1d6b606e445bad274
+    uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@f450e9219b0b77fa32ce27c5e9a9d9144cc577dd
     with:
       artifact-name: ${{ needs.build-and-test-linux.outputs.examples-artifact-name }}
       # Update next line if you need new version of proof producer
-      proof-producer-ref: ef8cd9152b4bec871e7efdc1d6b606e445bad274
+      proof-producer-ref: f450e9219b0b77fa32ce27c5e9a9d9144cc577dd
       refs: ${{ needs.handle-syncwith.outputs.prs-refs }}
       targets: ${{ needs.build-and-test-linux.outputs.prover-targets }}
 

bin/recursive_gen/src/main.cpp

@@ -106,6 +106,101 @@ std::optional<MarshallingType> decode_marshalling_from_file(
     return marshalled_data;
 }
 
+std::string add_filename_prefix(
+    const std::string& prefix,
+    const std::string& file_name
+) {
+    std::filesystem::path path(file_name);
+    std::filesystem::path parent_path = path.parent_path();
+    std::filesystem::path filename = path.filename();
+
+    std::string new_filename = prefix + filename.string();
+    std::filesystem::path new_path = parent_path / new_filename;
+
+    return new_path.string();
+}
+
+template<typename MarshallingType>
+std::optional<MarshallingType> decode_marshalling_from_different_column_types_files(
+    const std::string& assignment_table_file_name
+    //bool hex = false  // Used only for placeholder proof
+) {
+        std::ifstream iassignment_header;
+        std::ifstream iassignment_witness;
+        std::ifstream iassignment_pub_inp;
+        std::ifstream iassignment_constants;
+        std::ifstream iassignment_selectors;
+
+        iassignment_header.open(add_filename_prefix("header_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in);
+        if (!iassignment_header) {
+            std::cout << "Cannot open " << add_filename_prefix("header_", assignment_table_file_name) << std::endl;
+            return std::nullopt;
+        }
+        iassignment_witness.open(add_filename_prefix("witness_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in);
+        if (!iassignment_witness) {
+            std::cout << "Cannot open " << add_filename_prefix("witness_", assignment_table_file_name) << std::endl;
+            return std::nullopt;
+        }
+        iassignment_pub_inp.open(add_filename_prefix("pub_inp_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in);
+        if (!iassignment_pub_inp) {
+            std::cout << "Cannot open " << add_filename_prefix("pub_inp_", assignment_table_file_name) << std::endl;
+            return std::nullopt;
+        }
+        iassignment_constants.open(add_filename_prefix("constants_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in);
+        if (!iassignment_constants) {
+            std::cout << "Cannot open " << add_filename_prefix("constants_", assignment_table_file_name) << std::endl;
+            return std::nullopt;
+        }
+        iassignment_selectors.open(add_filename_prefix("selectors_", assignment_table_file_name), std::ios_base::binary | std::ios_base::in);
+        if (!iassignment_selectors) {
+            std::cout << "Cannot open " << add_filename_prefix("selectors_", assignment_table_file_name) << std::endl;
+            return std::nullopt;
+        }
+        std::vector<std::uint8_t> v;
+        iassignment_header.seekg(0, std::ios_base::end);
+        iassignment_witness.seekg(0, std::ios_base::end);
+        iassignment_pub_inp.seekg(0, std::ios_base::end);
+        iassignment_constants.seekg(0, std::ios_base::end);
+        iassignment_selectors.seekg(0, std::ios_base::end);
+
+        const auto header_size = iassignment_header.tellg();
+        const auto w_size = iassignment_witness.tellg();
+        const auto pi_size = iassignment_pub_inp.tellg();
+        const auto c_size = iassignment_constants.tellg();
+        const auto s_size = iassignment_selectors.tellg();
+
+        v.resize(header_size + w_size + pi_size + c_size + s_size);
+
+        iassignment_header.seekg(0, std::ios_base::beg);
+        iassignment_witness.seekg(0, std::ios_base::beg);
+        iassignment_pub_inp.seekg(0, std::ios_base::beg);
+        iassignment_constants.seekg(0, std::ios_base::beg);
+        iassignment_selectors.seekg(0, std::ios_base::beg);
+
+        iassignment_header.read(reinterpret_cast<char*>(v.data()),      header_size);
+        iassignment_witness.read(reinterpret_cast<char*>(v.data())    + header_size,  w_size);
+        iassignment_pub_inp.read(reinterpret_cast<char*>(v.data())    + header_size + w_size,  pi_size);
+        iassignment_constants.read(reinterpret_cast<char*>(v.data())  + header_size + w_size + pi_size,  c_size);
+        iassignment_selectors.read(reinterpret_cast<char*>(v.data())  + header_size + w_size + pi_size + c_size, s_size);
+
+        iassignment_header.close();
+        iassignment_witness.close();
+        iassignment_pub_inp.close();
+        iassignment_constants.close();
+        iassignment_selectors.close();
+
+        MarshallingType marshalled_data;
+        auto read_iter = v.begin();
+        auto status = marshalled_data.read(read_iter, v.size());
+
+        if (status != nil::marshalling::status_type::success) {
+            std::cerr << "Marshalled structure decoding failed" << std::endl;
+            return std::nullopt;
+        }
+        return marshalled_data;
+}
+
+
 template<typename BlueprintFieldType>
 struct ParametersPolicy {
     constexpr static const std::size_t WitnessColumns = WITNESS_COLUMNS;
@@ -414,7 +509,7 @@ int curve_dependent_main(
     public_input_sizes = (*constraint_system).public_input_sizes();
 
     if( vm.count("assignment-table") ){
-        auto marshalled_value = decode_marshalling_from_file<assignment_table_marshalling_type>(assignment_table_file_name);
+        auto marshalled_value = decode_marshalling_from_different_column_types_files<assignment_table_marshalling_type>(assignment_table_file_name);
         if (!marshalled_value) {
             return false;
         }

bin/transpiler/src/main.cpp

@@ -242,6 +242,43 @@ int main(int argc, char *argv[]) {
 
 }
 
+std::string add_filename_prefix(
+    const std::string& prefix,
+    const std::string& file_name
+) {
+    std::filesystem::path path(file_name);
+    std::filesystem::path parent_path = path.parent_path();
+    std::filesystem::path filename = path.filename();
+
+    std::string new_filename = prefix + filename.string();
+    std::filesystem::path new_path = parent_path / new_filename;
+
+    return new_path.string();
+}
+
+bool read_file_to_vector (
+    std::vector<std::uint8_t>& result_vector,
+    const std::string& prefix,
+    const std::string& assignment_table_file_name
+
+) {
+    std::cout << "assignment_table_file_name: " << assignment_table_file_name << "\n";
+    std::ifstream icolumn;
+    icolumn.open(add_filename_prefix(prefix, assignment_table_file_name), std::ios_base::binary | std::ios_base::in);
+    if (!icolumn) {
+        std::cout << "Cannot open " << add_filename_prefix("header_", assignment_table_file_name) << std::endl;
+        return false;
+    }
+    icolumn.seekg(0, std::ios_base::end);
+    const auto input_size = icolumn.tellg();
+    std::size_t old_size = result_vector.size();
+    result_vector.resize(old_size + input_size);
+    icolumn.seekg(0, std::ios_base::beg);
+    icolumn.read(reinterpret_cast<char*>(result_vector.data() + old_size), input_size);
+    icolumn.close();
+    return true;
+}
+
 template<typename BlueprintFieldType, bool is_multi_prover>
 int curve_dependent_main(
     boost::program_options::options_description options_desc,
@@ -361,23 +398,15 @@ int curve_dependent_main(
 
     AssignmentTableType assignment_table;
     {
-        std::ifstream iassignment;
-        iassignment.open(assignment_table_file_name, std::ios_base::binary | std::ios_base::in);
-        if (!iassignment) {
-            std::cout << "Cannot open " << assignment_table_file_name << std::endl;
-            return 1;
-        }
+
         std::vector<std::uint8_t> v;
-        iassignment.seekg(0, std::ios_base::end);
-        const auto fsize = iassignment.tellg();
-        v.resize(fsize);
-        iassignment.seekg(0, std::ios_base::beg);
-        iassignment.read(reinterpret_cast<char*>(v.data()), fsize);
-        if (!iassignment) {
-            std::cout << "Cannot parse input file " << assignment_table_file_name << std::endl;
-            return 1;
-        }
-        iassignment.close();
+
+        ASSERT(read_file_to_vector(v, "header_", assignment_table_file_name));
+        ASSERT(read_file_to_vector(v, "witness_", assignment_table_file_name));
+        ASSERT(read_file_to_vector(v, "pub_inp_", assignment_table_file_name));
+        ASSERT(read_file_to_vector(v, "constants_", assignment_table_file_name));
+        ASSERT(read_file_to_vector(v, "selectors_", assignment_table_file_name));
+
         table_value_marshalling_type marshalled_table_data;
         auto read_iter = v.begin();
         auto status = marshalled_table_data.read(read_iter, v.size());

examples/CMakeLists.txt

@@ -33,7 +33,11 @@ function(assign_ir)
     add_custom_target(${target}_generate_crct
             COMMAND $<TARGET_FILE:assigner>
             -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            ${minus_p} ${private_input_string}
             -c circuit_${target}.crct
+            -j table_pieces_${target}.json
+            -t assignment_${target}.tbl
             -e ${curve_type}
             --generate-type circuit
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -48,6 +52,7 @@ function(assign_ir)
             -i ${INPUTS_DIR}/${input}
             ${minus_p} ${private_input_string}
             -c circuit_${target}.crct
+            -j table_pieces_${target}.json
             -t assignment_${target}.tbl -e ${curve_type} --check
             --generate-type circuit-assignment
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -62,6 +67,7 @@ function(assign_ir)
             -i ${INPUTS_DIR}/${input}
             ${minus_p} ${private_input_string}
             -c circuit_${target}.crct
+            -j table_pieces_${target}.json
             -t assignment_${target}.tbl -e ${curve_type}
             --generate-type circuit-assignment
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -83,7 +89,35 @@ function(assign_ir)
         COMMAND_EXPAND_LISTS
         VERBATIM)
 
-    add_custom_target(${target}_generate_tbl_no_check
+    add_custom_target(${target}_generate_tbl_fast
+        COMMAND $<TARGET_FILE:assigner>
+            -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            ${minus_p} ${private_input_string}
+            -j table_pieces_${target}.json
+            -t assignment_${target}.tbl -e ${curve_type}
+            --generate-type assignment-fast
+            ${max_num_provers_flag} ${max_num_provers_amount}
+            ${arithmetization_flag} ${arithmetization_amount}
+        DEPENDS ${target} ${INPUTS_DIR}/${input} $<TARGET_FILE:assigner>
+        COMMAND_EXPAND_LISTS
+        VERBATIM)
+
+    add_custom_target(${target}_generate_tbl_fast_depends_on_crct
+        COMMAND $<TARGET_FILE:assigner>
+            -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            ${minus_p} ${private_input_string}
+            -j table_pieces_${target}.json
+            -t assignment_${target}.tbl -e ${curve_type}
+            --generate-type assignment-fast
+            ${max_num_provers_flag} ${max_num_provers_amount}
+            ${arithmetization_flag} ${arithmetization_amount}
+        DEPENDS ${target} ${INPUTS_DIR}/${input} ${target}_generate_crct $<TARGET_FILE:assigner>
+        COMMAND_EXPAND_LISTS
+        VERBATIM)
+
+        add_custom_target(${target}_generate_tbl_no_check
         COMMAND $<TARGET_FILE:assigner>
             -b ${binary_name}
             -i ${INPUTS_DIR}/${input}
@@ -99,6 +133,9 @@ function(assign_ir)
     add_custom_target(${target}_estimate_size
         COMMAND $<TARGET_FILE:assigner>
             -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            -j table_pieces_${target}.json
+            ${minus_p} ${private_input_string}
             -e ${curve_type}
             --generate-type size_estimation
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -120,15 +157,15 @@ function(assign_ir)
         VERBATIM)
 endfunction()
 
-function(gen_proof target curve_type provers_amount)
+function(gen_proof target curve_type provers_amount tbl_gen_speed)
     if(provers_amount EQUAL 0)
-        gen_single_proof(${target} ${curve_type} 0)
+        gen_single_proof(${target} ${curve_type} 0 ${tbl_gen_speed})
     else()
         add_custom_target(${target}_prove)
 
         foreach(prover_num RANGE 1 ${provers_amount})
             math(EXPR prover_num_minus_1 "${prover_num} - 1")
-            gen_single_proof(${target} ${curve_type} ${prover_num})
+            gen_single_proof(${target} ${curve_type} ${prover_num} ${tbl_gen_speed})
             add_dependencies(${target}_prove ${target}_prove${prover_num_minus_1})
         endforeach()
 
@@ -143,22 +180,27 @@ function(gen_proof target curve_type provers_amount)
 endfunction()
 
 
-function(gen_single_proof target curve_type provers_amount)
+function(gen_single_proof target curve_type provers_amount tbl_gen_speed)
     if(NOT provers_amount EQUAL 0)
         set(multi_prover_flag --multi-prover)
         math(EXPR prover_num "${provers_amount} - 1")
     else()
         set(prover_num "")
     endif()
 
+    set (depends_tbl ${target}_generate_both)
+    if (tbl_gen_speed STREQUAL "fast")
+        set (depends_tbl ${target}_generate_tbl_fast_depends_on_crct)
+    endif()
+
     add_custom_target(${target}_prove${prover_num}
         COMMAND $<TARGET_FILE:transpiler> -m gen-test-proof
             -c circuit_${target}.crct${prover_num}
             -t assignment_${target}.tbl${prover_num}
             -o transpiler_output_${target}${prover_num}
             -e ${curve_type}
             ${multi_prover_flag}
-            DEPENDS ${target}_generate_crct ${target}_generate_tbl $<TARGET_FILE:transpiler>
+            DEPENDS ${depends_tbl} $<TARGET_FILE:transpiler>
         COMMAND_EXPAND_LISTS
         VERBATIM)
 endfunction()