diff --git a/.github/workflows/check-proof-producer.yml b/.github/workflows/check-proof-producer.yml new file mode 100644 index 0000000000..6eae426273 --- /dev/null +++ b/.github/workflows/check-proof-producer.yml @@ -0,0 +1,243 @@ +name: Check Proof producer + +on: + workflow_call: + +env: + CAA_ARTIFACT_NAME: circuits-and-assignments + TO_ARTIFACT_NAME: transpiler-output + INTEGRATION_TESTING_TARGETS: | + arithmetics_cpp_example + polynomial_cpp_example + poseidon_cpp_example + merkle_tree_poseidon_cpp_example + uint_remainder_cpp + compare_eq_cpp + private_input_cpp + # uint_shift_left + # uint_bit_decomposition + # uint_bit_composition + +jobs: + prepare-targets: + name: Prepare targets strings + runs-on: ubuntu-22.04 + if: | + always() && !cancelled() + outputs: + evm-targets: ${{ steps.get-targets.outputs.evm-targets }} + prover-targets: ${{ steps.get-targets.outputs.prover-targets }} + steps: + - name: Set targets for integration testing + id: get-targets + run: | + targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {print "transpiler_output_" $1}') + echo "evm-targets<> $GITHUB_OUTPUT + echo "${targets_str}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + echo "prover-targets<> $GITHUB_OUTPUT + echo "${{ env.INTEGRATION_TESTING_TARGETS }}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + + get-zkllvm-run: + name: Get zkLLVM run to use artifacts from + if: | + always() && !cancelled() + runs-on: ubuntu-22.04 + outputs: + run-id: ${{ steps.get-run-id.outputs.run-id }} + steps: + - name: Get run ID of zkLLVM + id: get-run-id + env: + GH_TOKEN: ${{ github.token }} + run: | + zkllvm_ref="master" + zkllvm_repo="NilFoundation/zkLLVM" + + while read -r line; do + echo "$line" + if [[ $line == "${zkllvm_repo}:"* ]]; then + zkllvm_ref=$(echo "$line" | cut -d ' ' -f 2) + break + fi + done <<< "$prs_refs" + + if [[ $zkllvm_ref == refs/pull/* ]]; then + echo "Considering reference ${zkllvm_ref} a pr ref" + pr_number=${zkllvm_ref#refs/pull/} + pr_number=${pr_number%/merge} + sha=$(gh api repos/${zkllvm_repo}/pulls/$pr_number --jq '.head.sha') + elif [[ $zkllvm_ref == refs/tags/* ]]; then + echo "Considering reference ${zkllvm_ref} a tag" + tag=${zkllvm_ref#refs/tags/} + sha=$(gh api repos/${zkllvm_repo}/git/ref/tags/$tag --jq '.object.sha') + else + echo "Considering reference ${zkllvm_ref} a branch" + branch=${zkllvm_ref#refs/heads/} + # We can already fetch run_id here, but better fit to common approach with extra query by sha + sha=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?branch=${branch}&status=completed&per_page=1" \ + --jq '.workflow_runs[0].head_sha') + fi + + echo "Using head sha: ${sha}" + run_id=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?head_sha=${sha}&status=completed&per_page=1" \ + --jq '.workflow_runs[0].id') + if [ -z "${run_id}" ]; then + echo no run ID fetched + exit 1 + fi + echo "Run ID: ${run_id}" + for artifact in "${{ env.CAA_ARTIFACT_NAME }}" "${{ env.TO_ARTIFACT_NAME }}"; do + # Check if the artifact exists in the run + if ! gh run view ${run_id} --repo ${zkllvm_repo} | grep "$artifact"; then + echo "Artifact '$artifact' not found in run ${run_id}" + exit 1 + fi + done + echo "run-id=${run_id}" >> $GITHUB_OUTPUT + + build-and-generate-proofs: + name: Build prover, generate proofs for circuits + runs-on: [self-hosted, Linux, X64, aws_autoscaling] + needs: + - get-zkllvm-run + - prepare-targets + if: | + always() && !cancelled() && + (needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') && + (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') + outputs: + artifact-name: ${{ steps.artifact-name.outputs.merged }} + + steps: + # https://github.com/actions/checkout/issues/1552 + - name: Clean up after previous checkout + run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*; + + - name: Checkout Proof Producer + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Checkout submodules to specified refs + if: inputs.submodules-refs != '' + uses: NilFoundation/ci-cd/actions/recursive-checkout@v1.2.1 + with: + paths: | + ${{ github.workspace }}/** + !${{ github.workspace }}/ + !${{ github.workspace }}/**/.git/** + + - name: Run checks + run: nix build -L .?#packages.x86_64-linux.proof-producer + + - name: Set usefull strings + id: strings + run: | + echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" >> $GITHUB_OUTPUT + echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" + + - name: Download circuits and assignments artifact + uses: dawidd6/action-download-artifact@v3 + with: + repo: NilFoundation/zkLLVM + name: ${{ env.CAA_ARTIFACT_NAME }} + path: ${{ steps.strings.outputs.artifact-dir }} + run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + skip_unpack: true # It can't unpack such large files (>2Gb for some circuits) + + - name: Extract circuits and assignments artifact + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: | + unzip -o ${{ env.CAA_ARTIFACT_NAME }}.zip + + - name: List artifacts + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: find . + + - name: Make proofs for pairs + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: | + extra_args="" + if [[ "true" == "true" ]]; then + extra_args="--multithreaded " + fi + targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//') + echo "targets from input: ${targets_str}" + ${{ github.workspace }}/proof-producer/tests/make_proof_for_pairs.sh --use-nix ${extra_args} ${targets_str} + + - name: Download transpiler output artifact + id: download-to-artifact + uses: dawidd6/action-download-artifact@v3 + with: + repo: NilFoundation/zkLLVM + name: ${{ env.TO_ARTIFACT_NAME }} + path: ${{ steps.strings.outputs.artifact-dir }} + run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Merge proofs into transpiler output + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: | + copy_failed=0 + while read dir; do + base_name=${dir#./transpiler_output_} + + if [[ -d "$base_name" ]]; then + if ! cp "${base_name}/proof.bin" "${dir}/"; then + # fails if artifact contains dirs targets we didn't produce proofs for + echo "Failed to copy proof.bin to ${dir}" >&2 + copy_failed=1 + else + echo "proof.bin added to ${dir}" + fi + else + echo "Error: No matching directory found for ${dir}" >&2 + fi + done < <(find . -type d -name "transpiler_output_*") + + if [ $copy_failed -eq 1 ]; then + echo "One or more copy operations failed." >&2 + fi + + - name: Set aritfact name + id: artifact-name + run: | + echo "merged=transpiler-output-merged-proofs" >> $GITHUB_OUTPUT + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.merged }} + path: | + ${{ steps.strings.outputs.artifact-dir }}/transpiler_output_* + + # TODO: add package derivation to nix, upload its result + # - name: Upload .deb package + # if: matrix.cpp-compiler == 'clang++' && matrix.build-type == 'Release' + # uses: actions/upload-artifact@v4 + # with: + # name: proof-producer.deb + # path: | + # ${{ steps.strings.outputs.build-dir }}/proof-producer*.deb + + + verify-proof-producer-proofs: + name: Verify proof-producer proofs with EVM-placeholder + needs: + - build-and-generate-proofs + - prepare-targets + if: | + always() && !cancelled() && + (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') && + (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') + uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d + with: + artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }} + evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d + test-names: ${{ needs.prepare-targets.outputs.evm-targets }} diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 7f2f09e15c..093b840c31 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -29,3 +29,10 @@ jobs: always() && !cancelled() secrets: inherit + check-proof-producer: + name: Check Proof Producer + uses: ./.github/workflows/check-proof-producer.yml + if: | + always() && !cancelled() + secrets: inherit + diff --git a/flake.nix b/flake.nix index 346cbd3089..cde2b711ba 100644 --- a/flake.nix +++ b/flake.nix @@ -102,13 +102,35 @@ crypto3 = crypto3; }); + proof-producer = (pkgs.callPackage ./proof-producer/proof-producer.nix { + runTests = false; + enableDebug = false; + crypto3 = crypto3; + transpiler = transpiler; + parallel-crypto3 = parallel-crypto3; + }); + proof-producer-tests = (pkgs.callPackage ./proof-producer/proof-producer.nix { + runTests = true; + enableDebug = false; + crypto3 = crypto3; + transpiler = transpiler; + parallel-crypto3 = parallel-crypto3; + }); + proof-producer-debug-tests = (pkgs.callPackage ./proof-producer/proof-producer.nix { + enableDebug = true; + runTests = true; + crypto3 = crypto3; + transpiler = transpiler; + parallel-crypto3 = parallel-crypto3; + }); + # The "all" package will build all packages. Convenient for CI, # so that "nix build" will check that all packages are correct. # The packages that have no changes will not be rebuilt, and instead # fetched from the cache. all = pkgs.symlinkJoin { name = "all"; - paths = [ crypto3 evm-assigner zkevm-framework transpiler]; + paths = [ crypto3 evm-assigner zkevm-framework transpiler proof-producer]; }; default = all; }; @@ -167,15 +189,41 @@ crypto3 = crypto3-clang; }); + proof-producer-gcc = (pkgs.callPackage ./proof-producer/proof-producer.nix { + runTests = true; + enableDebug = false; + crypto3 = packages.crypto3; + transpiler = packages.transpiler; + parallel-crypto3 = packages.parallel-crypto3; + }); + proof-producer-clang = (pkgs.callPackage ./proof-producer/proof-producer.nix { + stdenv = pkgs.llvmPackages_18.stdenv; + runTests = true; + enableDebug = false; + crypto3 = crypto3-clang; + transpiler = transpiler-clang; + parallel-crypto3 = parallel-crypto3-clang; + }); + all-clang = pkgs.symlinkJoin { name = "all"; - paths = [ crypto3-clang parallel-crypto3-clang evm-assigner-clang transpiler-clang ]; + paths = [ crypto3-clang parallel-crypto3-clang evm-assigner-clang transpiler-clang proof-producer-clang ]; }; all-gcc = pkgs.symlinkJoin { name = "all"; - paths = [ crypto3-gcc parallel-crypto3-gcc evm-assigner-gcc zkevm-framework-gcc transpiler-gcc ]; + paths = [ crypto3-gcc parallel-crypto3-gcc evm-assigner-gcc zkevm-framework-gcc transpiler-gcc proof-producer-gcc ]; }; default = all-gcc; }; + apps = { + single-threaded = { + type = "app"; + program = "${self.packages.${system}.proof-producer}/bin/proof-producer-single-threaded"; + }; + default = { + type = "app"; + program = "${self.packages.${system}.default}/bin/proof-producer-multi-threaded"; + }; + }; })); } diff --git a/proof-producer/.pre-commit-config.yaml b/proof-producer/.pre-commit-config.yaml new file mode 100644 index 0000000000..309ff13298 --- /dev/null +++ b/proof-producer/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: 'v18.1.6' # Use the sha / tag you want to point at + hooks: + - id: clang-format diff --git a/proof-producer/CMakeLists.txt b/proof-producer/CMakeLists.txt new file mode 100644 index 0000000000..9eefb3270e --- /dev/null +++ b/proof-producer/CMakeLists.txt @@ -0,0 +1,82 @@ +#---------------------------------------------------------------------------# +# Copyright (c) 2022 Mikhail Komarov +# Copyright (c) 2022 Aleksei Moskvin +# Copyright (c) 2024 Iosif (x-mass) +# +# Distributed under the Boost Software License, Version 1.0 +# See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt +#---------------------------------------------------------------------------# + +cmake_minimum_required(VERSION 3.22 FATAL_ERROR) + +if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") + set(CMAKE_CXX_FLAGS "-ggdb -O0") + set(ZK_PLACEHOLDER_DEBUG_ENABLED TRUE) +endif() + +cmake_policy(SET CMP0042 NEW) +cmake_policy(SET CMP0028 NEW) +cmake_policy(SET CMP0057 NEW) + +find_package(CM REQUIRED) +include(CMConfig) +include(CMDeploy) +include(CMSetupVersion) + +cm_project(proof-producer WORKSPACE_NAME ${CMAKE_WORKSPACE_NAME} LANGUAGES CXX) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") + +find_package(crypto3 REQUIRED) +find_package(crypto3_transpiler REQUIRED) + +find_package(actor_zk REQUIRED) +find_package(actor_containers REQUIRED) +find_package(actor_core REQUIRED) +find_package(actor_math REQUIRED) + +# TODO: move thread to actor-core +find_package(Boost REQUIRED COMPONENTS filesystem log program_options thread) +if(ENABLE_TESTS) + find_package(Boost REQUIRED COMPONENTS unit_test_framework) +endif() + +# Read version from VERSION file +set(VERSION_FILE "${CMAKE_CURRENT_LIST_DIR}/VERSION") +file(READ ${VERSION_FILE} PROOF_PRODUCER_VERSION) +string(STRIP ${PROOF_PRODUCER_VERSION} PROOF_PRODUCER_VERSION) +# Remove leading 'v' from tag +string(REGEX REPLACE "^v" "" PROOF_PRODUCER_VERSION ${PROOF_PRODUCER_VERSION}) +if(NOT PROOF_PRODUCER_VERSION) + message(FATAL_ERROR + "Unable to retrive version from git or ${VERSION_FILE} file.") +endif() + +# Split numbers +string(REPLACE "-" "." PROOF_PRODUCER_VERSION_LIST ${PROOF_PRODUCER_VERSION}) +string(REPLACE "." ";" PROOF_PRODUCER_VERSION_LIST ${PROOF_PRODUCER_VERSION_LIST}) + +list(LENGTH PROOF_PRODUCER_VERSION_LIST VERSION_LIST_LENGHT) + +list(GET PROOF_PRODUCER_VERSION_LIST 0 PROOF_PRODUCER_VERSION_MAJOR) +list(GET PROOF_PRODUCER_VERSION_LIST 1 PROOF_PRODUCER_VERSION_MINOR) +list(GET PROOF_PRODUCER_VERSION_LIST 2 PROOF_PRODUCER_VERSION_PATCH) +if(VERSION_LIST_LENGHT GREATER 3) + list(GET PROOF_PRODUCER_VERSION_LIST 3 PROOF_PRODUCER_VERSION_INCREMENT) +endif() + +set(CPACK_GENERATOR DEB) +set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) +set(CPACK_DEBIAN_PACKAGE_MAINTAINER "=nil; Foundation") + +set(CPACK_PACKAGE_VERSION "${PROOF_PRODUCER_VERSION_MAJOR}.${PROOF_PRODUCER_VERSION_MINOR}.${PROOF_PRODUCER_VERSION_PATCH}") +if(PROOF_PRODUCER_VERSION_INCREMENT) + string(APPEND CPACK_PACKAGE_VERSION "-${PROOF_PRODUCER_VERSION_INCREMENT}") +endif() + +set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) + +add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/bin/proof-producer") + +include(CPack) diff --git a/proof-producer/README.md b/proof-producer/README.md new file mode 100644 index 0000000000..0ad37485a4 --- /dev/null +++ b/proof-producer/README.md @@ -0,0 +1,59 @@ +# Proof producer for the =nil; Proof Market +This repository contains the proof producer for the =nil; [Proof Market](https://proof.market/), which is a part of the =nil; [zkllvm toolchain](https://github.com/NilFoundation/zkLLVM) for zk-enabled applications development. + +# How to use + +The input for the proof producer is a circified version of the algorithm to be proven. This circified version of the algorithm is generated by the [zkllvm](https://raw.githubusercontent.com/NilFoundation/zkllvm) toolchain. + +Typically, you want to use the proof producer to participate in the =nil; Proof Market. In this case, you need to have a valid account on the =nil; Proof Market, which you can create through the [Proof Market web interface](https://proof.market/) or by using the [Proof Market CLI](https://github.com/NilFoundation/proof-market-toolchain/). + +# Installation + +All parts of the zkLLVM toolchain are distributed in form of deb packages. To install them, you need to add the =nil; repository to your systems package manager: + +```bash +echo 'deb [trusted=yes] http://deb.nil.foundation/ubuntu/ all main' >>/etc/apt/sources.list +apt update +``` + +Then, you can install the proof producer by running: + +```bash +apt install proof-producer +``` + +# Usage + +The proof producer is a command line tool. To see the list of available options, run: + +```bash +proof-generator --help +``` + +To produce a proof, you need to provide the proof producer with the file with the circuit definition and the assignment table with the values of the execution trace. You generate them from the [zkllvm examples](https://github.com/NilFoundation/zkLLVM) or download the existing ones using the [Proof Market CLI](https://github.com/NilFoundation/proof-market-toolchain/). + +When you have the circuit definition and the assignment table, you can produce a proof by running: + +```bash +proof-generator --circuit --assignment --proof +``` + +# Building from source +1. Install dependencies: + ``` + sudo apt-get install \ + build-essential \ + libsctp-dev \ + libssl-dev \ + libicu-dev \ + lsb-release \ + gnutls-dev \ + pkg-config + ``` + +2. Build with CMake: + ```mkdir build + cd build + cmake .. + make -j $(nrpoc) + ``` diff --git a/proof-producer/VERSION b/proof-producer/VERSION new file mode 100644 index 0000000000..1892b92676 --- /dev/null +++ b/proof-producer/VERSION @@ -0,0 +1 @@ +1.3.2 diff --git a/proof-producer/bin/proof-producer/CMakeLists.txt b/proof-producer/bin/proof-producer/CMakeLists.txt new file mode 100644 index 0000000000..ec905b2d8c --- /dev/null +++ b/proof-producer/bin/proof-producer/CMakeLists.txt @@ -0,0 +1,88 @@ +#---------------------------------------------------------------------------# +# Copyright (c) 2022 Mikhail Komarov +# Copyright (c) 2022 Aleksei Moskvin +# Copyright (c) 2023 Ilia Shirobokov +# Copyright (c) 2024 Iosif (x-mass) +# +# Distributed under the Boost Software License, Version 1.0 +# See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt +#---------------------------------------------------------------------------# + +include(CMDeploy) +include(CMSetupVersion) + +if (CPACK_PACKAGE_VERSION) + add_compile_definitions(PROOF_GENERATOR_VERSION=${CPACK_PACKAGE_VERSION}) +endif() + +option(ZK_PLACEHOLDER_PROFILING_ENABLED "Build with placeholder profiling" FALSE) +option(ZK_PLACEHOLDER_DEBUG_ENABLED "Build with placeholder testing inside" FALSE) + +if(ZK_PLACEHOLDER_PROFILING) + add_definitions(-DZK_PLACEHOLDER_PROFILING_ENABLED) +endif() + +if(ZK_PLACEHOLDER_DEBUG) + add_definitions(-DZK_PLACEHOLDER_DEBUG_ENABLED) +endif() + +# Function to setup common properties for a target +function(setup_proof_generator_target) + set(options "") + set(oneValueArgs TARGET_NAME) + set(multiValueArgs ADDITIONAL_DEPENDENCIES) + + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + add_executable(${ARG_TARGET_NAME} + src/arg_parser.cpp + src/main.cpp + ) + + # Make sure to add these dependencies before the others, since for multi-threaded version dependency on + # actor-zk must come first. + if(ARG_ADDITIONAL_DEPENDENCIES) + foreach(lib IN LISTS ARG_ADDITIONAL_DEPENDENCIES) + target_link_libraries(${ARG_TARGET_NAME} ${lib}) + endforeach() + endif() + + set_target_properties(${ARG_TARGET_NAME} PROPERTIES + LINKER_LANGUAGE CXX + EXPORT_NAME ${ARG_TARGET_NAME} + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED TRUE) + + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_compile_options(${ARG_TARGET_NAME} PRIVATE "-fconstexpr-steps=2147483647") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options(${ARG_TARGET_NAME} PRIVATE "-fconstexpr-ops-limit=4294967295") + endif () + + target_include_directories(${ARG_TARGET_NAME} PUBLIC + $ + $ + ) + + target_link_libraries(${ARG_TARGET_NAME} + crypto3::all + + crypto3::transpiler + + Boost::filesystem + Boost::log + Boost::program_options + Boost::thread + ) +endfunction() + +# Declare single-threaded target +set(SINGLE_THREADED_TARGET "${CURRENT_PROJECT_NAME}-single-threaded") +setup_proof_generator_target(TARGET_NAME ${SINGLE_THREADED_TARGET}) + +# Declare multi-threaded target +set(MULTI_THREADED_TARGET "${CURRENT_PROJECT_NAME}-multi-threaded") +setup_proof_generator_target(TARGET_NAME ${MULTI_THREADED_TARGET} ADDITIONAL_DEPENDENCIES actor::zk) + +install(TARGETS ${SINGLE_THREADED_TARGET} ${MULTI_THREADED_TARGET} RUNTIME DESTINATION bin) diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp new file mode 100644 index 0000000000..fe00a924e1 --- /dev/null +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp @@ -0,0 +1,59 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#ifndef PROOF_GENERATOR_ARG_PARSER_HPP +#define PROOF_GENERATOR_ARG_PARSER_HPP + +#include + +#include +#include + +#include +#include + +namespace nil { + namespace proof_generator { + + using CurvesVariant = + typename tuple_to_variant::type>::type; + using HashesVariant = + typename tuple_to_variant::type>::type; + + struct ProverOptions { + boost::filesystem::path proof_file_path = "proof.bin"; + boost::filesystem::path json_file_path = "proof.json"; + boost::filesystem::path preprocessed_common_data_path = "preprocessed_common_data.dat"; + boost::filesystem::path circuit_file_path; + boost::filesystem::path assignment_table_file_path; + boost::log::trivial::severity_level log_level = boost::log::trivial::severity_level::info; + bool skip_verification = false; + bool verification_only = false; + CurvesVariant elliptic_curve_type = type_identity{}; + HashesVariant hash_type = type_identity>{}; + + std::size_t lambda = 9; + std::size_t grind = 69; + std::size_t expand_factor = 2; + std::size_t max_quotient_chunks = 0; + }; + + std::optional parse_args(int argc, char* argv[]); + + } // namespace proof_generator +} // namespace nil + +#endif // PROOF_GENERATOR_ARG_PARSER_HPP diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/arithmetization_params.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/arithmetization_params.hpp new file mode 100644 index 0000000000..4e432459cb --- /dev/null +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/arithmetization_params.hpp @@ -0,0 +1,49 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#ifndef PROOF_GENERATOR_ARITHMETIZATION_PARAMS_HPP +#define PROOF_GENERATOR_ARITHMETIZATION_PARAMS_HPP + +#include +#include + +#include +#include +#include +#include +#include + +#include + +namespace nil { + namespace proof_generator { + + using CurveTypes = std::tuple; + + using HashTypes = std::tuple< + nil::crypto3::hashes::keccak_1600<256>, + nil::crypto3::hashes::sha2<256>, + nil::crypto3::hashes::poseidon> + // Add more hashes as needed. + >; + + } // namespace proof_generator +} // namespace nil + +#endif // PROOF_GENERATOR_ARITHMETIZATION_PARAMS_HPP diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/file_operations.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/file_operations.hpp new file mode 100644 index 0000000000..43e496c0cd --- /dev/null +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/file_operations.hpp @@ -0,0 +1,192 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#ifndef PROOF_GENERATOR_FILE_OPERATIONS_HPP +#define PROOF_GENERATOR_FILE_OPERATIONS_HPP + +#include +#include +#include +#include +#include + +#include +#include + +namespace nil { + namespace proof_generator { + inline bool is_valid_path(const std::string& path) { + if (path.length() >= PATH_MAX) { + BOOST_LOG_TRIVIAL(error) << path << ": file path is too long. Maximum allowed length is " << PATH_MAX + << " characters."; + return false; + } + + if (boost::filesystem::path(path).filename().string().length() >= FILENAME_MAX) { + BOOST_LOG_TRIVIAL(error) << path << ": file name is too long. Maximum allowed length is " + << FILENAME_MAX << " characters."; + return false; + } + return true; + } + + inline bool can_read_from_file(const std::string& path) { + if (!is_valid_path(path)) { + return false; + } + + std::ifstream file(path, std::ios::in); + bool can_read = file.is_open(); + file.close(); + return can_read; + } + + inline bool can_write_to_file(const std::string& path) { + if (!is_valid_path(path)) { + return false; + } + + if (boost::filesystem::exists(path)) { + std::ofstream file(path, std::ios::out | std::ios::app); + bool can_write = file.is_open(); + return can_write; + } else { + boost::filesystem::path boost_path = boost::filesystem::absolute(path); + boost::filesystem::path parent_dir = boost_path.parent_path(); + if (parent_dir.empty()) { + BOOST_LOG_TRIVIAL(error) << "Proof parent dir is empty. Seems like you " + "are passing an empty string."; + return false; + } + if (!boost::filesystem::exists(parent_dir)) { + BOOST_LOG_TRIVIAL(error) << boost_path << ": proof parent dir does not exist. Create it first."; + return false; + } + std::string temp_file_name = (parent_dir / "temp_file_to_test_write_permission").string(); + std::ofstream temp_file(temp_file_name, std::ios::out); + bool can_write = temp_file.is_open(); + temp_file.close(); + + if (can_write) { + boost::filesystem::remove(temp_file_name); + } + return can_write; + } + } + + template + std::optional open_file(const std::string& path, std::ios_base::openmode mode) { + StreamType file(path, mode); + if (!file.is_open()) { + BOOST_LOG_TRIVIAL(error) << "Unable to open file: " << path; + return std::nullopt; + } + + return file; + } + + std::optional> read_file_to_vector(const std::string& path) { + + auto file = open_file(path, std::ios_base::in | std::ios::binary | std::ios::ate); + if (!file.has_value()) { + return std::nullopt; + } + + std::ifstream& stream = file.value(); + std::streamsize fsize = stream.tellg(); + stream.seekg(0, std::ios::beg); + std::vector v(static_cast(fsize)); + + stream.read(reinterpret_cast(v.data()), fsize); + + if (stream.fail()) { + BOOST_LOG_TRIVIAL(error) << "Error occurred during reading file " << path; + return std::nullopt; + } + + return v; + } + + bool write_vector_to_file(const std::vector& vector, const std::string& path) { + + auto file = open_file(path, std::ios_base::out | std::ios_base::binary); + if (!file.has_value()) { + return false; + } + + std::ofstream& stream = file.value(); + stream.write(reinterpret_cast(vector.data()), vector.size()); + + if (stream.fail()) { + BOOST_LOG_TRIVIAL(error) << "Error occured during writing file " << path; + return false; + } + + return true; + } + + // HEX data format is not efficient, we will remove it later + std::optional> read_hex_file_to_vector(const std::string& path) { + auto file = open_file(path, std::ios_base::in); + if (!file.has_value()) { + return std::nullopt; + } + + std::ifstream& stream = file.value(); + std::vector result; + std::string line; + while (std::getline(stream, line)) { + if (line.rfind("0x", 0) == 0 && line.length() >= 3) { + for (size_t i = 2; i < line.length(); i += 2) { + std::string hex_string = line.substr(i, 2); + uint8_t byte = static_cast(std::stoul(hex_string, nullptr, 16)); + result.push_back(byte); + } + } else { + BOOST_LOG_TRIVIAL(error) << "File contains non-hex string"; + return std::nullopt; + } + } + + return result; + } + + bool write_vector_to_hex_file(const std::vector& vector, const std::string& path) { + auto file = open_file(path, std::ios_base::out); + if (!file.has_value()) { + return false; + } + + std::ofstream& stream = file.value(); + + stream << "0x" << std::hex; + for (auto it = vector.cbegin(); it != vector.cend(); ++it) { + stream << std::setfill('0') << std::setw(2) << std::right << int(*it); + } + stream << std::dec; + + if (stream.fail()) { + BOOST_LOG_TRIVIAL(error) << "Error occurred during writing to file " << path; + return false; + } + + return true; + } + + } // namespace proof_generator +} // namespace nil + +#endif // PROOF_GENERATOR_FILE_OPERATIONS_HPP diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/meta_utils.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/meta_utils.hpp new file mode 100644 index 0000000000..144b808203 --- /dev/null +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/meta_utils.hpp @@ -0,0 +1,110 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#ifndef PROOF_GENERATOR_META_UTILS_HPP +#define PROOF_GENERATOR_META_UTILS_HPP + +#include +#include + +namespace nil { + namespace proof_generator { + + // C++20 has it inside std:: + template + struct type_identity { + using type = T; + }; + + template + struct to_type_identity { + using type = type_identity; + }; + + template class MetaFun> + struct transform_tuple; + + template class MetaFun> + struct transform_tuple, MetaFun> { + using type = std::tuple::type...>; + }; + + template + struct tuple_to_variant; + + template + struct tuple_to_variant> { + using type = std::variant; + }; + + template + struct variant_to_tuple; + + template + struct variant_to_tuple> { + using type = std::tuple; + }; + + // Find passed value among constexpr array, passes found index to function + // template param. Example: + // constexpr std::array arr = {'a', 'b'}; // It could be some + // complex struct with defined `==` operator auto func = []() + // { + // std::cout << arr[N]; + // }; + // char inp; + // std::cin >> inp; + // generate_templates_from_array_for_runtime_check(inp, func); + template + constexpr void generate_templates_from_array_for_runtime_check(RuntimeT runtime_value, Func function) { + if constexpr (Idx < std::size(ConstexprArray)) { + if (ConstexprArray[Idx] == runtime_value) { + function.template operator()(); + } else { + generate_templates_from_array_for_runtime_check( + runtime_value, + function + ); + } + } else { + throw std::runtime_error("Runtime value not found among constexpr array elements."); + } + } + + // Takes variant value and forward its inner type to function template. + // Example: + // using Variant = std::variant; + // Variant var; + // var = 69; + // auto func = []() { + // std::cout << typeid(template_type).name(); + // }; + // pass_variant_type_to_template_func(var, func); + template + void pass_variant_type_to_template_func(const Variant& variant, Func&& func) { + std::visit( + [&func](auto&& arg) { + using CurrentType = std::decay_t; + func.template operator()(); + }, + variant + ); + } + + } // namespace proof_generator +} // namespace nil + +#endif // PROOF_GENERATOR_META_UTILS_HPP diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/non_type_arithmetization_params.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/non_type_arithmetization_params.hpp new file mode 100644 index 0000000000..b3e1106212 --- /dev/null +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/non_type_arithmetization_params.hpp @@ -0,0 +1,58 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#ifndef PROOF_GENERATOR_NON_TYPE_ARITHMETIZATION_PARAMS_HPP +#define PROOF_GENERATOR_NON_TYPE_ARITHMETIZATION_PARAMS_HPP + +#include + +namespace nil { + namespace proof_generator { + + // Need this class to be derived into actual params, so we could overload + // read/write operators for parsing. + class SizeTParam { + public: + SizeTParam() = default; // lexical_cast wants it + constexpr SizeTParam(std::size_t value) + : value_(value) { + } + + constexpr operator std::size_t() const { + return value_; + } + + bool operator==(const SizeTParam& other) const { + return value_ == other.value_; + } + + private: + std::size_t value_; + }; + + class LambdaParam : public SizeTParam { + public: + using SizeTParam::SizeTParam; + }; + class GrindParam : public SizeTParam { + public: + using SizeTParam::SizeTParam; + }; + + } // namespace proof_generator +} // namespace nil + +#endif // PROOF_GENERATOR_NON_TYPE_ARITHMETIZATION_PARAMS_HPP diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp new file mode 100644 index 0000000000..ac6fbcad4c --- /dev/null +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp @@ -0,0 +1,365 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2021 Mikhail Komarov +// Copyright (c) 2021 Nikita Kaskov +// Copyright (c) 2022-2023 Ilia Shirobokov +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#ifndef PROOF_GENERATOR_ASSIGNER_PROOF_HPP +#define PROOF_GENERATOR_ASSIGNER_PROOF_HPP + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace nil { + namespace proof_generator { + namespace detail { + template + std::optional decode_marshalling_from_file( + const boost::filesystem::path& path, + bool hex = false + ) { + const auto v = hex ? read_hex_file_to_vector(path.c_str()) : read_file_to_vector(path.c_str()); + if (!v.has_value()) { + return std::nullopt; + } + + MarshallingType marshalled_data; + auto read_iter = v->begin(); + auto status = marshalled_data.read(read_iter, v->size()); + if (status != nil::marshalling::status_type::success) { + BOOST_LOG_TRIVIAL(error) << "Marshalled structure decoding failed"; + return std::nullopt; + } + return marshalled_data; + } + + template + bool encode_marshalling_to_file( + const boost::filesystem::path& path, + const MarshallingType& data_for_marshalling, + bool hex = false + ) { + std::vector v; + v.resize(data_for_marshalling.length(), 0x00); + auto write_iter = v.begin(); + nil::marshalling::status_type status = data_for_marshalling.write(write_iter, v.size()); + if (status != nil::marshalling::status_type::success) { + BOOST_LOG_TRIVIAL(error) << "Marshalled structure encoding failed"; + return false; + } + + return hex ? write_vector_to_hex_file(v, path.c_str()) : write_vector_to_file(v, path.c_str()); + } + + std::vector generate_random_step_list(const std::size_t r, const int max_step) { + using Distribution = std::uniform_int_distribution; + static std::random_device random_engine; + + std::vector step_list; + std::size_t steps_sum = 0; + while (steps_sum != r) { + if (r - steps_sum <= max_step) { + while (r - steps_sum != 1) { + step_list.emplace_back(r - steps_sum - 1); + steps_sum += step_list.back(); + } + step_list.emplace_back(1); + steps_sum += step_list.back(); + } else { + step_list.emplace_back(Distribution(1, max_step)(random_engine)); + steps_sum += step_list.back(); + } + } + return step_list; + } + } // namespace detail + + template + class Prover { + public: + Prover( + boost::filesystem::path circuit_file_name, + boost::filesystem::path preprocessed_common_data_file_name, + boost::filesystem::path assignment_table_file_name, + boost::filesystem::path proof_file, + boost::filesystem::path json_file, + std::size_t lambda, + std::size_t expand_factor, + std::size_t max_q_chunks, + std::size_t grind + ) + : circuit_file_(circuit_file_name) + , preprocessed_common_data_file_(preprocessed_common_data_file_name) + , assignment_table_file_(assignment_table_file_name) + , proof_file_(proof_file) + , json_file_(json_file) + , lambda_(lambda) + , expand_factor_(expand_factor) + , max_quotient_chunks_(max_q_chunks) + , grind_(grind) { + } + + bool generate_to_file(bool skip_verification) { + if (!nil::proof_generator::can_write_to_file(proof_file_.string())) { + BOOST_LOG_TRIVIAL(error) << "Can't write to file " << proof_file_; + return false; + } + + prepare_for_operation(); + + BOOST_ASSERT(public_preprocessed_data_); + BOOST_ASSERT(private_preprocessed_data_); + BOOST_ASSERT(table_description_); + BOOST_ASSERT(constraint_system_); + BOOST_ASSERT(lpc_scheme_); + BOOST_ASSERT(fri_params_); + + BOOST_LOG_TRIVIAL(info) << "Generating proof..."; + Proof proof = nil::crypto3::zk::snark::placeholder_prover::process( + *public_preprocessed_data_, + *private_preprocessed_data_, + *table_description_, + *constraint_system_, + *lpc_scheme_ + ); + BOOST_LOG_TRIVIAL(info) << "Proof generated"; + + if (skip_verification) { + BOOST_LOG_TRIVIAL(info) << "Skipping proof verification"; + } else { + if (!verify(proof)) { + return false; + } + } + + BOOST_LOG_TRIVIAL(info) << "Writing proof to " << proof_file_; + auto filled_placeholder_proof = + nil::crypto3::marshalling::types::fill_placeholder_proof(proof, *fri_params_); + bool res = nil::proof_generator::detail::encode_marshalling_to_file( + proof_file_, + filled_placeholder_proof, + true + ); + if (res) { + BOOST_LOG_TRIVIAL(info) << "Proof written"; + } + + BOOST_LOG_TRIVIAL(info) << "Writing json proof to " << json_file_; + auto output_file = open_file(json_file_.string(), std::ios_base::out); + if (!output_file) + return res; + + (*output_file) << nil::blueprint::recursive_verifier_generator< + PlaceholderParams, + nil::crypto3::zk::snark::placeholder_proof, + typename nil::crypto3::zk::snark::placeholder_public_preprocessor< + BlueprintField, + PlaceholderParams>::preprocessed_data_type::common_data_type>( + *table_description_ + ) + .generate_input(*public_inputs_, proof, constraint_system_->public_input_sizes()); + output_file->close(); + + return res; + } + + bool verify_from_file() { + prepare_for_operation(); + using ProofMarshalling = nil::crypto3::marshalling::types:: + placeholder_proof, Proof>; + BOOST_LOG_TRIVIAL(info) << "Reading proof from file"; + auto marshalled_proof = detail::decode_marshalling_from_file(proof_file_, true); + if (!marshalled_proof) { + return false; + } + bool res = + verify(nil::crypto3::marshalling::types::make_placeholder_proof(*marshalled_proof + )); + if (res) { + BOOST_LOG_TRIVIAL(info) << "Proof verified"; + } + return res; + } + + bool save_preprocessed_common_data_to_file() { + BOOST_LOG_TRIVIAL(info) << "Writing preprocessed common data to file..."; + using Endianness = nil::marshalling::option::big_endian; + using TTypeBase = nil::marshalling::field_type; + using CommonData = typename PublicPreprocessedData::preprocessed_data_type::common_data_type; + auto marshalled_common_data = + nil::crypto3::marshalling::types::fill_placeholder_common_data( + public_preprocessed_data_->common_data + ); + bool res = nil::proof_generator::detail::encode_marshalling_to_file( + preprocessed_common_data_file_, + marshalled_common_data + ); + if (res) { + BOOST_LOG_TRIVIAL(info) << "Preprocessed common data written"; + } + return res; + } + + private: + using BlueprintField = typename CurveType::base_field_type; + using LpcParams = nil::crypto3::zk::commitments::list_polynomial_commitment_params; + using Lpc = nil::crypto3::zk::commitments::list_polynomial_commitment; + using LpcScheme = typename nil::crypto3::zk::commitments::lpc_commitment_scheme; + using CircuitParams = nil::crypto3::zk::snark::placeholder_circuit_params; + using PlaceholderParams = nil::crypto3::zk::snark::placeholder_params; + using Proof = nil::crypto3::zk::snark::placeholder_proof; + using PublicPreprocessedData = typename nil::crypto3::zk::snark:: + placeholder_public_preprocessor::preprocessed_data_type; + using PrivatePreprocessedData = typename nil::crypto3::zk::snark:: + placeholder_private_preprocessor::preprocessed_data_type; + using ConstraintSystem = nil::crypto3::zk::snark::plonk_constraint_system; + using TableDescription = nil::crypto3::zk::snark::plonk_table_description; + using Endianness = nil::marshalling::option::big_endian; + using FriParams = typename Lpc::fri_type::params_type; + using Column = nil::crypto3::zk::snark::plonk_column; + using AssignmentTable = nil::crypto3::zk::snark::plonk_table; + + bool verify(const Proof& proof) const { + BOOST_LOG_TRIVIAL(info) << "Verifying proof..."; + bool verification_result = + nil::crypto3::zk::snark::placeholder_verifier::process( + public_preprocessed_data_->common_data, + proof, + *table_description_, + *constraint_system_, + *lpc_scheme_ + ); + + if (verification_result) { + BOOST_LOG_TRIVIAL(info) << "Proof is verified"; + } else { + BOOST_LOG_TRIVIAL(error) << "Proof verification failed"; + } + + return verification_result; + } + + bool prepare_for_operation() { + using BlueprintField = typename CurveType::base_field_type; + using TTypeBase = nil::marshalling::field_type; + using ConstraintMarshalling = + nil::crypto3::marshalling::types::plonk_constraint_system; + + { + auto marshalled_value = detail::decode_marshalling_from_file(circuit_file_); + if (!marshalled_value) { + return false; + } + constraint_system_.emplace( + nil::crypto3::marshalling::types::make_plonk_constraint_system( + *marshalled_value + ) + ); + } + + using TableValueMarshalling = + nil::crypto3::marshalling::types::plonk_assignment_table; + auto marshalled_table = + detail::decode_marshalling_from_file(assignment_table_file_); + if (!marshalled_table) { + return false; + } + auto [table_description, assignment_table] = + nil::crypto3::marshalling::types::make_assignment_table( + *marshalled_table + ); + + public_inputs_.emplace(assignment_table.public_inputs()); + table_description_.emplace(table_description); + + // Lambdas and grinding bits should be passed threw preprocessor directives + std::size_t table_rows_log = std::ceil(std::log2(table_description_->rows_amount)); + + fri_params_.emplace(FriParams(1, table_rows_log, lambda_, expand_factor_)); + lpc_scheme_.emplace(*fri_params_); + + BOOST_LOG_TRIVIAL(info) << "Preprocessing public data"; + public_preprocessed_data_.emplace( + nil::crypto3::zk::snark::placeholder_public_preprocessor:: + process( + *constraint_system_, + assignment_table.move_public_table(), + *table_description_, + *lpc_scheme_, + max_quotient_chunks_ + ) + ); + + BOOST_LOG_TRIVIAL(info) << "Preprocessing private data"; + private_preprocessed_data_.emplace( + nil::crypto3::zk::snark::placeholder_private_preprocessor:: + process(*constraint_system_, assignment_table.move_private_table(), *table_description_) + ); + return true; + } + + const boost::filesystem::path circuit_file_; + const boost::filesystem::path preprocessed_common_data_file_; + const boost::filesystem::path assignment_table_file_; + const boost::filesystem::path proof_file_; + const boost::filesystem::path json_file_; + const std::size_t expand_factor_; + const std::size_t max_quotient_chunks_; + const std::size_t lambda_; + const std::size_t grind_; + + // All set on prepare_for_operation() + std::optional public_preprocessed_data_; + std::optional private_preprocessed_data_; + std::optional public_inputs_; + std::optional table_description_; + std::optional constraint_system_; + std::optional fri_params_; + std::optional lpc_scheme_; + }; + + } // namespace proof_generator +} // namespace nil + +#endif // PROOF_GENERATOR_ASSIGNER_PROOF_HPP diff --git a/proof-producer/bin/proof-producer/src/arg_parser.cpp b/proof-producer/bin/proof-producer/src/arg_parser.cpp new file mode 100644 index 0000000000..06bf3b9f29 --- /dev/null +++ b/proof-producer/bin/proof-producer/src/arg_parser.cpp @@ -0,0 +1,209 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#include "nil/proof-generator/arg_parser.hpp" + +#include "nil/proof-generator/arithmetization_params.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +namespace nil { + namespace proof_generator { + namespace po = boost::program_options; + + void check_exclusive_options(const po::variables_map& vm, const std::vector& opts) { + std::vector found_opts; + for (const auto& opt : opts) { + if (vm.count(opt) && !vm[opt].defaulted()) { + found_opts.push_back(opt); + } + } + if (found_opts.size() > 1) { + throw std::logic_error("Conflicting options: " + boost::algorithm::join(found_opts, " and ")); + } + } + + template + po::typed_value* make_defaulted_option(T& variable) { + return po::value(&variable)->default_value(variable); + } + + std::optional parse_args(int argc, char* argv[]) { + po::options_description options("Nil; Proof Generator Options"); + // Declare a group of options that will be + // allowed only on command line + po::options_description generic("CLI options"); + // clang-format off + generic.add_options() + ("help,h", "Produce help message") + ("version,v", "Print version string") + ("config,c", po::value(), "Config file path"); + // clang-format on + + ProverOptions prover_options; + + // Declare a group of options that will be + // allowed both on command line and in + // config file + po::options_description config( + "Configuration", + /*line_length=*/120, + /*min_description_length=*/60 + ); + // clang-format off + auto options_appender = config.add_options() + ("proof,p", make_defaulted_option(prover_options.proof_file_path), "Output proof file") + ("json,j", make_defaulted_option(prover_options.json_file_path), "JSON proof file") + ("common-data,d", make_defaulted_option(prover_options.preprocessed_common_data_path), "Output preprocessed common data file") + ("circuit", po::value(&prover_options.circuit_file_path)->required(), "Circuit input file") + ("assignment-table,t", po::value(&prover_options.assignment_table_file_path)->required(), "Assignment table input file") + ("log-level,l", make_defaulted_option(prover_options.log_level), "Log level (trace, debug, info, warning, error, fatal)") + ("elliptic-curve-type,e", make_defaulted_option(prover_options.elliptic_curve_type), "Elliptic curve type (pallas)") + ("hash-type", make_defaulted_option(prover_options.hash_type), "Hash type (keccak, poseidon, sha256)") + ("lambda-param", make_defaulted_option(prover_options.lambda), "Lambda param (9)") + ("grind-param", make_defaulted_option(prover_options.grind), "Grind param (69)") + ("expand-factor,x", make_defaulted_option(prover_options.expand_factor), "Expand factor") + ("max-quotient-chunks,q", make_defaulted_option(prover_options.max_quotient_chunks), "Maximum quotient polynomial parts amount") + ("skip-verification", po::bool_switch(&prover_options.skip_verification), "Skip generated proof verifying step") + ("verification-only", po::bool_switch(&prover_options.verification_only), "Read proof for verification instead of writing to it"); + // clang-format on + po::options_description cmdline_options("nil; Proof Producer"); + cmdline_options.add(generic).add(config); + + po::variables_map vm; + try { + po::store(parse_command_line(argc, argv, cmdline_options), vm); + } catch (const po::validation_error& e) { + std::cerr << e.what() << std::endl; + std::cout << cmdline_options << std::endl; + throw e; + } + + if (vm.count("help")) { + std::cout << cmdline_options << std::endl; + return std::nullopt; + } + + if (vm.count("version")) { +#ifdef PROOF_GENERATOR_VERSION +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + std::cout << TOSTRING(PROOF_GENERATOR_VERSION) << std::endl; +#undef STRINGIFY +#undef TOSTRING +#else + std::cout << "undefined" << std::endl; +#endif + return std::nullopt; + } + + // Parse configuration file. Args from CLI will not be overwritten + if (vm.count("config")) { + std::ifstream ifs(vm["config"].as().c_str()); + if (ifs) { + store(parse_config_file(ifs, config), vm); + } else { + throw std::runtime_error("Cannot open config file: " + vm["config"].as()); + } + } + + // Calling notify(vm) after handling no-op cases prevent parser from alarming + // about absence of required args + try { + notify(vm); + } catch (const po::required_option& e) { + std::cerr << e.what() << std::endl; + std::cout << cmdline_options << std::endl; + throw e; + } + + try { + check_exclusive_options(vm, {"verification-only", "skip-verification"}); + } catch (const std::logic_error& e) { + std::cerr << e.what() << std::endl; + std::cout << cmdline_options << std::endl; + throw e; + } + + return prover_options; + } + +// Here we have generators of read and write operators for options holding +// types. Don't forget to adjust help message when add new type - name mapping. +// Examples below. +#define GENERATE_WRITE_OPERATOR(TYPE_TO_STRING_LINES, VARIANT_TYPE) \ + std::ostream& operator<<(std::ostream& strm, const VARIANT_TYPE& variant) { \ + strm << std::visit( \ + [&strm](auto&& arg) -> std::string { \ + using SelectedType = std::decay_t; \ + TYPE_TO_STRING_LINES \ + strm.setstate(std::ios_base::failbit); \ + return ""; \ + }, \ + variant \ + ); \ + return strm; \ + } +#define TYPE_TO_STRING(TYPE, NAME) \ + if constexpr (std::is_same_v>) \ + return NAME; + +#define GENERATE_READ_OPERATOR(STRING_TO_TYPE_LINES, VARIANT_TYPE) \ + std::istream& operator>>(std::istream& strm, VARIANT_TYPE& variant) { \ + std::string str; \ + strm >> str; \ + auto l = [&str, &strm]() -> VARIANT_TYPE { \ + STRING_TO_TYPE_LINES \ + strm.setstate(std::ios_base::failbit); \ + return VARIANT_TYPE(); \ + }; \ + variant = l(); \ + return strm; \ + } +#define STRING_TO_TYPE(TYPE, NAME) \ + if (NAME == str) \ + return type_identity{}; + +#define CURVE_TYPES X(nil::crypto3::algebra::curves::pallas, "pallas") +#define X(type, name) TYPE_TO_STRING(type, name) + GENERATE_WRITE_OPERATOR(CURVE_TYPES, CurvesVariant) +#undef X +#define X(type, name) STRING_TO_TYPE(type, name) + GENERATE_READ_OPERATOR(CURVE_TYPES, CurvesVariant) +#undef X + +#define HASH_TYPES \ + X(nil::crypto3::hashes::keccak_1600<256>, "keccak") \ + X(nil::crypto3::hashes::poseidon>, \ + "poseidon") \ + X(nil::crypto3::hashes::sha2<256>, "sha256") +#define X(type, name) TYPE_TO_STRING(type, name) + GENERATE_WRITE_OPERATOR(HASH_TYPES, HashesVariant) +#undef X +#define X(type, name) STRING_TO_TYPE(type, name) + GENERATE_READ_OPERATOR(HASH_TYPES, HashesVariant) +#undef X + + } // namespace proof_generator +} // namespace nil diff --git a/proof-producer/bin/proof-producer/src/main.cpp b/proof-producer/bin/proof-producer/src/main.cpp new file mode 100644 index 0000000000..15d37be392 --- /dev/null +++ b/proof-producer/bin/proof-producer/src/main.cpp @@ -0,0 +1,95 @@ +//---------------------------------------------------------------------------// +// Copyright (c) 2018-2021 Mikhail Komarov +// Copyright (c) 2022 Aleksei Moskvin +// Copyright (c) 2022 Ilia Shirobokov +// Copyright (c) 2024 Iosif (x-mass) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//---------------------------------------------------------------------------// + +#include +#include +#include + +#include +#include +#include + +#undef B0 + +using namespace nil::proof_generator; + +template +int run_prover(const nil::proof_generator::ProverOptions& prover_options) { + auto prover_task = [&] { + auto prover = nil::proof_generator::Prover( + prover_options.circuit_file_path, + prover_options.preprocessed_common_data_path, + prover_options.assignment_table_file_path, + prover_options.proof_file_path, + prover_options.json_file_path, + prover_options.lambda, + prover_options.expand_factor, + prover_options.max_quotient_chunks, + prover_options.grind + ); + bool prover_result; + try { + prover_result = prover_options.verification_only ? prover.verify_from_file() + : prover.generate_to_file(prover_options.skip_verification) + && prover.save_preprocessed_common_data_to_file(); + } catch (const std::exception& e) { + BOOST_LOG_TRIVIAL(error) << e.what(); + return 1; + } + return prover_result ? 0 : 1; + }; + return prover_task(); +} + +// We could either make lambdas for generating Cartesian products of templates, +// but this would lead to callback hell. Instead, we declare extra function for +// each factor. Last declared function starts the chain. +template +int hash_wrapper(const ProverOptions& prover_options) { + int ret; + auto run_prover_wrapper_void = [&prover_options, &ret]() { + using HashType = typename HashTypeIdentity::type; + ret = run_prover(prover_options); + }; + pass_variant_type_to_template_func(prover_options.hash_type, run_prover_wrapper_void); + return ret; +} + +int curve_wrapper(const ProverOptions& prover_options) { + int ret; + auto curves_wrapper_void = [&prover_options, &ret]() { + using CurveType = typename CurveTypeIdentity::type; + ret = hash_wrapper(prover_options); + }; + pass_variant_type_to_template_func(prover_options.elliptic_curve_type, curves_wrapper_void); + return ret; +} + +int initial_wrapper(const ProverOptions& prover_options) { + return curve_wrapper(prover_options); +} + +int main(int argc, char* argv[]) { + std::optional prover_options = nil::proof_generator::parse_args(argc, argv); + if (!prover_options) { + // Action has already taken a place (help, version, etc.) + return 0; + } + return initial_wrapper(*prover_options); +} diff --git a/proof-producer/proof-producer.nix b/proof-producer/proof-producer.nix new file mode 100644 index 0000000000..a964fed356 --- /dev/null +++ b/proof-producer/proof-producer.nix @@ -0,0 +1,43 @@ +{ lib, + stdenv, + ninja, + pkg-config, + cmake, + crypto3, + parallel-crypto3, + transpiler, + boost, + gdb, + cmake_modules, + enableDebugging, + enableDebug ? false, + runTests ? false, + }: +let + inherit (lib) optional; +in stdenv.mkDerivation { + name = "Proof-producer"; + + src = lib.sourceByRegex ./. [ ".*" ]; + + nativeBuildInputs = [ cmake ninja pkg-config ] ++ (lib.optional (!stdenv.isDarwin) gdb); + + # enableDebugging will keep debug symbols in boost + propagatedBuildInputs = [ (if enableDebug then (enableDebugging boost) else boost) ]; + + buildInputs = [cmake_modules crypto3 parallel-crypto3 transpiler ]; + + cmakeFlags = + [ + "-DCMAKE_INSTALL_PREFIX=${placeholder "out"}" + (if enableDebug then "-DCMAKE_BUILD_TYPE=Debug" else "-DCMAKE_BUILD_TYPE=Release") + "-G Ninja" + ]; + + doCheck = runTests; + + shellHook = '' + PS1="\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ " + echo "Welcome to Proof-producer development environment!" + ''; +} diff --git a/proof-producer/tests/make_proof_for_pairs.sh b/proof-producer/tests/make_proof_for_pairs.sh new file mode 100755 index 0000000000..5af16aa379 --- /dev/null +++ b/proof-producer/tests/make_proof_for_pairs.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +color_red() { echo -e "\033[31m$1\033[0m"; } +color_green() { echo -e "\033[32m$1\033[0m"; } + +get_script_dir() { + echo "$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +} + +make_proof_for_pair() { + local tbl_file=$1 + local crct_file="${tbl_file%assignment.tbl}circuit.crct" + local relative_tbl_path="$(realpath --relative-to="$base_dir" "$tbl_file")" + local proof_dir="${output_dir}/$(dirname ${relative_tbl_path})" + + local proof_generator_binary + if [ "$use_nix" = true ]; then + proof_generator_binary="nix run ${script_dir}/..?submodules=1#single-threaded --" + if [ "$use_multithreaded" = true ]; then + proof_generator_binary="nix run ${script_dir}/..?submodules=1# --" + fi + else + proof_generator_binary="${script_dir}/../build/bin/proof-generator/proof-generator-single-threaded" + if [ "$use_multithreaded" = true ]; then + proof_generator_binary="${script_dir}/../build/bin/proof-generator/proof-generator-multi-threaded" + fi + fi + + if [ -f "$crct_file" ]; then + mkdir -p "$proof_dir" # Ensure the output directory exists + echo -n "Processing $tbl_file and $crct_file (proof will be at $proof_dir; binary name: $proof_generator_binary): " + if $proof_generator_binary -t "$tbl_file" --circuit "$crct_file" --proof "$proof_dir/proof.bin" ${args_to_forward[@]}; then + color_green "success" + else + color_red "failed" + return 1 + fi + else + color_red "${crct_file} file not found for $tbl_file" + return 1 + fi +} + +clean_up() { + echo "Cleaning up proof files..." + find "$output_dir" -name 'proof' -type f -exec rm {} + +} + +parse_args() { + base_dir="." + output_dir="" + use_multithreaded=false + use_nix=false + args_to_forward=() + targets=() + + while [ "$#" -gt 0 ]; do + case "$1" in + --base-dir) + base_dir="$2" + shift 2 + ;; + --output-dir) + output_dir="$2" + shift 2 + ;; + --multithreaded) + use_multithreaded=true + shift + ;; + --use-nix) + use_nix=true + shift + ;; + --) + shift + while [ "$#" -gt 0 ]; do + args_to_forward+=("$1") # Collect remaining arguments + shift + done + ;; + *) + targets+=("$1") # Add remaining arguments as targets + shift + ;; + esac + done +} + +exit_code=0 +script_dir=$(get_script_dir) +clean=false +parse_args "$@" + +if [ -z "$output_dir" ]; then + echo "Output directory not specified. Using default." + output_dir=$base_dir +fi + +process_directory() { + local dir=$1 + while read tbl_file; do + if ! make_proof_for_pair "$tbl_file"; then + exit_code=1 + fi + done < <(find "$dir" -name 'assignment.tbl') +} + +# If targets are specified +if [ ${#targets[@]} -gt 0 ]; then + for target in "${targets[@]}"; do + process_directory "$base_dir/$target" + done +else + process_directory "$base_dir" +fi + +# Clean up if needed +if [ "$clean" = true ]; then + clean_up +fi + +exit $exit_code diff --git a/transpiler/CMakeLists.txt b/transpiler/CMakeLists.txt index 9b9ffc8dc1..b5bb934b81 100644 --- a/transpiler/CMakeLists.txt +++ b/transpiler/CMakeLists.txt @@ -22,10 +22,9 @@ if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") set(CMAKE_CXX_FLAGS "-ggdb -O0") endif() +cm_workspace(crypto3) cm_project(transpiler WORKSPACE_NAME ${CMAKE_WORKSPACE_NAME} LANGUAGES ASM C CXX) -include(CMDeploy) - cm_setup_version(VERSION 0.1.0 PREFIX ${CMAKE_WORKSPACE_NAME}_${CURRENT_PROJECT_NAME}) find_package(crypto3 REQUIRED) @@ -45,7 +44,7 @@ target_include_directories(${CMAKE_WORKSPACE_NAME}_${CURRENT_PROJECT_NAME} INTER target_link_libraries(${CMAKE_WORKSPACE_NAME}_${CURRENT_PROJECT_NAME} INTERFACE crypto3::all ${Boost_LIBRARIES}) - +include(CMDeploy) cm_deploy(TARGETS ${CMAKE_WORKSPACE_NAME}_${CURRENT_PROJECT_NAME} INCLUDE include NAMESPACE ${CMAKE_WORKSPACE_NAME}::)