wip

Add partial proof stage

Added partial prove step #13

Added merge-proof action #13

wip

theta power is saving now #13

rm debug prints

crypto3/libs/zk/include/nil/crypto3/zk/commitments/polynomial/lpc.hpp

@@ -352,8 +352,9 @@ namespace nil {
                     }
 
                     // Computes and returns the maximal power of theta used to compute the value of Combined_Q.
-                    std::size_t compute_theta_power_for_combined_Q() const {
+                    std::size_t compute_theta_power_for_combined_Q() {
                         std::size_t theta_power = 0;
+                        this->eval_polys();
                         this->build_points_map();
 
                         auto points = this->get_unique_points();
@@ -371,8 +372,9 @@ namespace nil {
                         }
 
                         for (std::size_t i: this->_z.get_batches()) {
-                            if (!_batch_fixed.at(i))
+                            if (!_batch_fixed[i]) {
                                 continue;
+                            }
 
                             theta_power += this->_z.get_batch_size(i);
                         }

crypto3/libs/zk/include/nil/crypto3/zk/snark/systems/plonk/placeholder/proof.hpp

@@ -29,6 +29,7 @@
 #define CRYPTO3_ZK_PLONK_PLACEHOLDER_PROOF_HPP
 
 #include <map>
+#include <vector>
 
 namespace nil {
     namespace crypto3 {

crypto3/libs/zk/include/nil/crypto3/zk/snark/systems/plonk/placeholder/prover.hpp

@@ -90,14 +90,15 @@ namespace nil {
                     constexpr static const std::size_t permutation_parts = 3;
                     constexpr static const std::size_t lookup_parts = 6;
                     constexpr static const std::size_t f_parts = 8;
-              public:
+
+                public:
 
                     static inline placeholder_proof<FieldType, ParamsType> process(
                         const typename public_preprocessor_type::preprocessed_data_type &preprocessed_public_data,
                         typename private_preprocessor_type::preprocessed_data_type preprocessed_private_data,
                         const plonk_table_description<FieldType> &table_description,
                         const plonk_constraint_system<FieldType> &constraint_system,
-                        commitment_scheme_type commitment_scheme,
+                        const commitment_scheme_type& commitment_scheme,
                         bool skip_commitment_scheme_eval_proofs = false
                     ) {
                         auto prover = placeholder_prover<FieldType, ParamsType>(
@@ -207,17 +208,20 @@ namespace nil {
                         }
                         transcript(_proof.commitments[QUOTIENT_BATCH]);
 
+                        // 8. Run evaluation proofs
+                        _proof.eval_proof.challenge = transcript.template challenge<FieldType>();
+                        generate_evaluation_points();
                         if (!_skip_commitment_scheme_eval_proofs) {
-                            // 8. Run evaluation proofs
-                            _proof.eval_proof.challenge = transcript.template challenge<FieldType>();
-                            generate_evaluation_points();
-
                             _proof.eval_proof.eval_proof = _commitment_scheme.proof_eval(transcript);
                         }
 
                         return _proof;
                     }
 
+                    commitment_scheme_type& get_commitment_scheme() {
+                        return _commitment_scheme;
+                    }
+
                 private:
                     std::vector<polynomial_dfs_type> quotient_polynomial_split_dfs() {
                         PROFILE_SCOPE("quotient_polynomial_split_dfs");

parallel-crypto3/libs/parallel-zk/include/nil/crypto3/zk/snark/systems/plonk/placeholder/prover.hpp

@@ -208,9 +208,8 @@ namespace nil {
                         transcript(_proof.commitments[QUOTIENT_BATCH]);
 
                         // 8. Run evaluation proofs
+                        _proof.eval_proof.challenge = transcript.template challenge<FieldType>();
                         if (!_skip_commitment_scheme_eval_proofs) {
-                            // 8. Run evaluation proofs
-                            _proof.eval_proof.challenge = transcript.template challenge<FieldType>();
                             generate_evaluation_points();
 
                             _proof.eval_proof.eval_proof = _commitment_scheme.proof_eval(transcript);

proof-producer/CMakeLists.txt

@@ -26,6 +26,26 @@ include(CMSetupVersion)
 
 cm_project(proof-producer WORKSPACE_NAME ${CMAKE_WORKSPACE_NAME} LANGUAGES CXX)
 
+# This is useful due to some build systems (Ninja in particular) are piping
+# compiler output and compiler switches it's output to plain text
+option (FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." FALSE)
+if (${FORCE_COLORED_OUTPUT})
+    if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+       add_compile_options (-fdiagnostics-color=always)
+    elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+       add_compile_options (-fcolor-diagnostics)
+    endif ()
+endif ()
+
+# The file compile_commands.json is generated in build directory, so LSP could
+# pick it up and guess all include paths, defines and other stuff.
+# If Nix is used, LSP could not guess the locations of implicit include
+# directories, so we need to include them explicitly.
+if(CMAKE_EXPORT_COMPILE_COMMANDS)
+  set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES 
+      ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
+endif()
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
 
 find_package(crypto3 REQUIRED)

proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp

@@ -18,6 +18,7 @@
 #define PROOF_GENERATOR_ARG_PARSER_HPP
 
 #include <optional>
+#include <string>
 
 #include <boost/filesystem/path.hpp>
 #include <boost/log/trivial.hpp>
@@ -43,7 +44,12 @@ namespace nil {
             boost::filesystem::path circuit_file_path;
             boost::filesystem::path assignment_table_file_path;
             boost::filesystem::path assignment_description_file_path;
+            boost::filesystem::path challenge_file_path;
+            boost::filesystem::path theta_power_file_path;
             std::vector<boost::filesystem::path> input_challenge_files;
+            std::vector<boost::filesystem::path> partial_proof_files;
+            std::vector<boost::filesystem::path> aggregated_proof_files;
+            boost::filesystem::path last_proof_file;
             boost::filesystem::path aggregated_challenge_file = "aggregated_challenge.dat";
             boost::filesystem::path combined_Q_polynomial_file = "combined_Q.dat";
             std::size_t combined_Q_starting_power;

proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp

@@ -122,7 +122,8 @@ namespace nil {
                     {"verify", ProverStage::VERIFY},
                     {"generate-aggregated-challenge", ProverStage::GENERATE_AGGREGATED_CHALLENGE},
                     {"partial-prove", ProverStage::PARTIAL_PROVE},
-                    {"compute-combined-Q", ProverStage::COMPUTE_COMBINED_Q}
+                    {"compute-combined-Q", ProverStage::COMPUTE_COMBINED_Q},
+                    {"merge-proofs", ProverStage::MERGE_PROOFS},
                 };
                 auto it = stage_map.find(stage);
                 if (it == stage_map.end()) {
@@ -174,6 +175,9 @@ namespace nil {
             bool generate_to_file(
                     boost::filesystem::path proof_file_,
                     boost::filesystem::path json_file_,
+                    std::optional<boost::filesystem::path> challenge_file_,
+                    std::optional<boost::filesystem::path> theta_power_file,
+                    bool partial_proof,
                     bool skip_verification) {
                 if (!nil::proof_generator::can_write_to_file(proof_file_.string())) {
                     BOOST_LOG_TRIVIAL(error) << "Can't write to file " << proof_file_;
@@ -187,16 +191,17 @@ namespace nil {
                 BOOST_ASSERT(lpc_scheme_);
 
                 BOOST_LOG_TRIVIAL(info) << "Generating proof...";
-                Proof proof = nil::crypto3::zk::snark::placeholder_prover<BlueprintField, PlaceholderParams>::process(
-                    *public_preprocessed_data_,
-                    *private_preprocessed_data_,
-                    *table_description_,
-                    *constraint_system_,
-                    *lpc_scheme_
-                );
+                auto prover = nil::crypto3::zk::snark::placeholder_prover<BlueprintField, PlaceholderParams>(
+                        *public_preprocessed_data_,
+                        *private_preprocessed_data_,
+                        *table_description_,
+                        *constraint_system_,
+                        *lpc_scheme_,
+                        partial_proof);
+                Proof proof = prover.process();
                 BOOST_LOG_TRIVIAL(info) << "Proof generated";
 
-                if (skip_verification) {
+                if (skip_verification || partial_proof) {
                     BOOST_LOG_TRIVIAL(info) << "Skipping proof verification";
                 } else {
                     if (!verify(proof)) {
@@ -234,6 +239,50 @@ namespace nil {
                                       .generate_input(*public_inputs_, proof, constraint_system_->public_input_sizes());
                 output_file->close();
 
+                if (partial_proof) {
+                    if (!challenge_file_) {
+                        BOOST_LOG_TRIVIAL(error) << "Challenge output file is not set.";
+                        return false;
+                    }
+                    if (!theta_power_file) {
+                        BOOST_LOG_TRIVIAL(error) << "Theta power file is not set.";
+                        return false;
+                    }
+                    BOOST_LOG_TRIVIAL(info) << "Writing challenge";
+                    using challenge_marshalling_type =
+                        nil::crypto3::marshalling::types::field_element<
+                        TTypeBase, typename BlueprintField::value_type>;
+
+                    challenge_marshalling_type marshalled_challenge(proof.eval_proof.challenge);
+
+                    bool res =
+                        detail::encode_marshalling_to_file<challenge_marshalling_type>(
+                            *challenge_file_,
+                            marshalled_challenge);
+                    if (res) {
+                        BOOST_LOG_TRIVIAL(info) << "Challenge written.";
+                    } else {
+                        BOOST_LOG_TRIVIAL(error) << "Failed to write challenge to file.";
+                    }
+
+                    auto commitment_scheme = prover.get_commitment_scheme();
+
+                    commitment_scheme.state_commited(crypto3::zk::snark::FIXED_VALUES_BATCH);
+                    commitment_scheme.state_commited(crypto3::zk::snark::VARIABLE_VALUES_BATCH);
+                    commitment_scheme.state_commited(crypto3::zk::snark::PERMUTATION_BATCH);
+                    commitment_scheme.state_commited(crypto3::zk::snark::QUOTIENT_BATCH);
+                    commitment_scheme.state_commited(crypto3::zk::snark::LOOKUP_BATCH);
+                    commitment_scheme.mark_batch_as_fixed(crypto3::zk::snark::FIXED_VALUES_BATCH);
+
+                    commitment_scheme.set_fixed_polys_values(common_data_->commitment_scheme_data);
+
+                    std::size_t theta_power = commitment_scheme.compute_theta_power_for_combined_Q();
+
+                    auto output_file = open_file<std::ofstream>(theta_power_file->string(), std::ios_base::out);
+                    (*output_file) << theta_power << std::endl;
+                    output_file->close();
+                }
+
                 return res;
             }
 
@@ -420,6 +469,7 @@ namespace nil {
                     );
                 table_description_.emplace(table_description);
                 assignment_table_.emplace(std::move(assignment_table));
+                public_inputs_.emplace(assignment_table_->public_inputs());
                 return true;
             }
 
@@ -588,6 +638,45 @@ namespace nil {
                 return save_poly_to_file(combined_Q, output_combined_Q_file); 
             }
 
+            bool merge_proofs(
+                const std::vector<boost::filesystem::path> &partial_proof_files,
+                const std::vector<boost::filesystem::path> &aggregated_proof_files,
+                const boost::filesystem::path &last_proof_file,
+                const boost::filesystem::path &merged_proof_file)
+            {
+                nil::crypto3::zk::snark::placeholder_aggregated_proof<BlueprintField, PlaceholderParams>
+                    merged_proof;
+
+                for(auto const& partial_proof_file: partial_proof_files) {
+                    using ProofMarshalling = nil::crypto3::marshalling::types::
+                        placeholder_proof<nil::marshalling::field_type<Endianness>, Proof>;
+
+                    BOOST_LOG_TRIVIAL(info) << "Reading partial proof from file \"" << partial_proof_file << "\"";
+                    auto marshalled_proof = detail::decode_marshalling_from_file<ProofMarshalling>(partial_proof_file, true);
+                    if (!marshalled_proof) {
+                        return false;
+                    }
+                    auto partial_proof = nil::crypto3::marshalling::types::make_placeholder_proof<Endianness, Proof>(*marshalled_proof);
+                    merged_proof.partial_proofs.emplace_back(partial_proof);
+                }
+
+                for(auto const& aggregated_proof_file: aggregated_proof_files) {
+
+                    /* TODO: Need marshalling for initial_proofs (lpc_proof_type) */
+                    BOOST_LOG_TRIVIAL(info) << "Reading aggregated part proof from file \"" << aggregated_proof_file << "\"";
+                    // merged_proof.aggregated_proof.intial_proofs_per_prover.emplace_back(initial_proof);
+                }
+
+                /* TODO: Need marshalling for top-level proof, (fri_proof_type) */
+                BOOST_LOG_TRIVIAL(info) << "Reading single round part proof from file \"" << last_proof_file << "\"";
+                // merged_proof.fri_proof = ...
+
+
+                BOOST_LOG_TRIVIAL(info) << "Writing merged proof to \"" << merged_proof_file << "\"";
+
+                return true;
+            }
+
         private:
             const std::size_t expand_factor_;
             const std::size_t max_quotient_chunks_;

proof-producer/bin/proof-producer/src/arg_parser.cpp

@@ -91,12 +91,23 @@ namespace nil {
                 ("max-quotient-chunks,q", make_defaulted_option(prover_options.max_quotient_chunks), "Maximum quotient polynomial parts amount")
                 ("input-challenge-files,u", po::value<std::vector<boost::filesystem::path>>(&prover_options.input_challenge_files)->multitoken(),
                  "Input challenge files. Used with 'generate-aggregated-challenge' stage.")
+                ("challenge-file", po::value<boost::filesystem::path>(&prover_options.challenge_file_path),
+                 "Input challenge files. Used with 'generate-aggregated-challenge' stage.")
+                ("theta-power-file", po::value<boost::filesystem::path>(&prover_options.theta_power_file_path),
+                 "File to output theta power. Used by main prover to arrange starting powers of Q")
                 ("aggregated-challenge-file", po::value<boost::filesystem::path>(&prover_options.aggregated_challenge_file),
                  "Aggregated challenge file. Used with 'generate-aggregated-challenge' stage")
                 ("combined-Q-polynomial-file", po::value<boost::filesystem::path>(&prover_options.combined_Q_polynomial_file),
                  "File containing the polynomial combined-Q, generated on a single prover.")
                 ("combined-Q-starting-power", po::value<std::size_t>(&prover_options.combined_Q_starting_power),
-                 "The starting power for combined-Q polynomial for the current prover.");
+                 "The starting power for combined-Q polynomial for the current prover.")
+                ("partial-proof", po::value<std::vector<boost::filesystem::path>>(&prover_options.partial_proof_files)->multitoken(),
+                 "Partial proofs. Used with 'merge-proofs' stage.")
+                ("aggregated-proof", po::value<std::vector<boost::filesystem::path>>(&prover_options.aggregated_proof_files)->multitoken(),
+                 "Parts of aggregated proof. Used with 'merge-proofs' stage.")
+                ("last-proof", po::value<boost::filesystem::path>(&prover_options.last_proof_file)->multitoken(),
+                 "Last proof of aggregated proof. Used with 'merge-proofs' stage.")
+                ;
 
             // clang-format on
             po::options_description cmdline_options("nil; Proof Producer");

proof-producer/bin/proof-producer/src/main.cpp

@@ -50,7 +50,10 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) {
                         prover.generate_to_file(
                             prover_options.proof_file_path,
                             prover_options.json_file_path,
-                            false/*don't skip verification*/) &&
+                            std::nullopt,
+                            std::nullopt,
+                            false, /* partial proof */
+                            false/*don't skip verification*/) && 
                         prover.save_preprocessed_common_data_to_file(prover_options.preprocessed_common_data_path) &&
                         prover.save_public_preprocessed_data_to_file(prover_options.preprocessed_public_data_path) &&
                         prover.save_commitment_state_to_file(prover_options.commitment_scheme_state_path);
@@ -76,6 +79,26 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) {
                         prover.generate_to_file(
                             prover_options.proof_file_path,
                             prover_options.json_file_path,
+                            std::nullopt,
+                            std::nullopt,
+                            false, /*partial proof*/
+                            true/*skip verification*/);
+                    break;
+                case nil::proof_generator::detail::ProverStage::PARTIAL_PROVE:
+                    // Load preprocessed data from file and generate the proof.
+                    prover_result =
+                        prover.read_circuit(prover_options.circuit_file_path) &&
+                        prover.read_assignment_table(prover_options.assignment_table_file_path) &&
+                        prover.read_public_preprocessed_data_from_file(prover_options.preprocessed_public_data_path) &&
+                        prover.read_preprocessed_common_data_from_file(prover_options.preprocessed_common_data_path) &&
+                        prover.read_commitment_scheme_from_file(prover_options.commitment_scheme_state_path) &&
+                        prover.preprocess_private_data() &&
+                        prover.generate_to_file(
+                            prover_options.proof_file_path,
+                            prover_options.json_file_path,
+                            prover_options.challenge_file_path,
+                            prover_options.theta_power_file_path,
+                            true, /* partial proof */
                             true/*skip verification*/);
                     break;
                 case nil::proof_generator::detail::ProverStage::VERIFY:
@@ -91,9 +114,19 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) {
                             prover_options.input_challenge_files,
                             prover_options.aggregated_challenge_file
                         );
+                    break;
+                case nil::proof_generator::detail::ProverStage::MERGE_PROOFS:
+                    prover_result =
+                        prover.merge_proofs(
+                            prover_options.partial_proof_files,
+                            prover_options.aggregated_proof_files,
+                            prover_options.last_proof_file,
+                            prover_options.proof_file_path);
+                    break;
             }
         } catch (const std::exception& e) {
             BOOST_LOG_TRIVIAL(error) << e.what();
+            throw e;
             return 1;
         }
         return prover_result ? 0 : 1;