Implement CI for proof-producer

.github/workflows/check-proof-producer.yml

@@ -0,0 +1,243 @@
+name: Check Proof producer
+
+on:
+  workflow_call:
+
+env:
+  CAA_ARTIFACT_NAME: circuits-and-assignments
+  TO_ARTIFACT_NAME: transpiler-output
+  INTEGRATION_TESTING_TARGETS: |
+    arithmetics_cpp_example
+    polynomial_cpp_example
+    poseidon_cpp_example
+    merkle_tree_poseidon_cpp_example
+    uint_remainder_cpp
+    compare_eq_cpp
+    private_input_cpp
+  # uint_shift_left
+  # uint_bit_decomposition
+  # uint_bit_composition
+
+jobs:
+  prepare-targets:
+    name: Prepare targets strings
+    runs-on: ubuntu-22.04
+    if: |
+      always() && !cancelled()
+    outputs:
+      evm-targets: ${{ steps.get-targets.outputs.evm-targets }}
+      prover-targets: ${{ steps.get-targets.outputs.prover-targets }}
+    steps:
+      - name: Set targets for integration testing
+        id: get-targets
+        run: |
+          targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {print "transpiler_output_" $1}')
+          echo "evm-targets<<EOF" >> $GITHUB_OUTPUT
+          echo "${targets_str}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+          echo "prover-targets<<EOF" >> $GITHUB_OUTPUT
+          echo "${{ env.INTEGRATION_TESTING_TARGETS }}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+
+  get-zkllvm-run:
+    name: Get zkLLVM run to use artifacts from
+    if: |
+      always() && !cancelled()
+    runs-on: ubuntu-22.04
+    outputs:
+      run-id: ${{ steps.get-run-id.outputs.run-id }}
+    steps:
+      - name: Get run ID of zkLLVM
+        id: get-run-id
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          zkllvm_ref="master"
+          zkllvm_repo="NilFoundation/zkLLVM"
+
+          while read -r line; do
+            echo "$line"
+            if [[ $line == "${zkllvm_repo}:"* ]]; then
+              zkllvm_ref=$(echo "$line" | cut -d ' ' -f 2)
+              break
+            fi
+          done <<< "$prs_refs"
+
+          if [[ $zkllvm_ref == refs/pull/* ]]; then
+            echo "Considering reference ${zkllvm_ref} a pr ref"
+            pr_number=${zkllvm_ref#refs/pull/}
+            pr_number=${pr_number%/merge}
+            sha=$(gh api repos/${zkllvm_repo}/pulls/$pr_number --jq '.head.sha')
+          elif [[ $zkllvm_ref == refs/tags/* ]]; then
+            echo "Considering reference ${zkllvm_ref} a tag"
+            tag=${zkllvm_ref#refs/tags/}
+            sha=$(gh api repos/${zkllvm_repo}/git/ref/tags/$tag --jq '.object.sha')
+          else
+            echo "Considering reference ${zkllvm_ref} a branch"
+            branch=${zkllvm_ref#refs/heads/}
+            # We can already fetch run_id here, but better fit to common approach with extra query by sha
+            sha=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?branch=${branch}&status=completed&per_page=1" \
+              --jq '.workflow_runs[0].head_sha')
+          fi
+
+          echo "Using head sha: ${sha}"
+          run_id=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?head_sha=${sha}&status=completed&per_page=1" \
+              --jq '.workflow_runs[0].id')
+          if [ -z "${run_id}" ]; then
+            echo no run ID fetched
+            exit 1
+          fi
+          echo "Run ID: ${run_id}"
+          for artifact in "${{ env.CAA_ARTIFACT_NAME }}" "${{ env.TO_ARTIFACT_NAME }}"; do
+              # Check if the artifact exists in the run
+              if ! gh run view ${run_id} --repo ${zkllvm_repo} | grep "$artifact"; then
+                  echo "Artifact '$artifact' not found in run ${run_id}"
+                  exit 1
+              fi
+          done
+          echo "run-id=${run_id}" >> $GITHUB_OUTPUT
+
+  build-and-generate-proofs:
+    name: Build prover, generate proofs for circuits
+    runs-on: [self-hosted, Linux, X64, aws_autoscaling]
+    needs:
+      - get-zkllvm-run
+      - prepare-targets
+    if: |
+      always() && !cancelled() &&
+      (needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') &&
+      (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped')
+    outputs:
+      artifact-name: ${{ steps.artifact-name.outputs.merged }}
+
+    steps:
+      # https://github.com/actions/checkout/issues/1552
+      - name: Clean up after previous checkout
+        run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*;
+
+      - name: Checkout Proof Producer
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+
+      - name: Checkout submodules to specified refs
+        if: inputs.submodules-refs != ''
+        uses: NilFoundation/ci-cd/actions/[email protected]
+        with:
+          paths: |
+            ${{ github.workspace }}/**
+            !${{ github.workspace }}/
+            !${{ github.workspace }}/**/.git/**
+
+      - name: Run checks
+        run: nix build -L .?#packages.x86_64-linux.proof-producer
+
+      - name: Set usefull strings
+        id: strings
+        run: |
+          echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" >> $GITHUB_OUTPUT
+          echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)"
+
+      - name: Download circuits and assignments artifact
+        uses: dawidd6/action-download-artifact@v3
+        with:
+          repo: NilFoundation/zkLLVM
+          name: ${{ env.CAA_ARTIFACT_NAME }}
+          path: ${{ steps.strings.outputs.artifact-dir }}
+          run_id: ${{ needs.get-zkllvm-run.outputs.run-id }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_unpack: true  # It can't unpack such large files (>2Gb for some circuits)
+
+      - name: Extract circuits and assignments artifact
+        working-directory: ${{ steps.strings.outputs.artifact-dir }}
+        run: |
+          unzip -o ${{ env.CAA_ARTIFACT_NAME }}.zip
+
+      - name: List artifacts
+        working-directory: ${{ steps.strings.outputs.artifact-dir }}
+        run: find .
+
+      - name: Make proofs for pairs
+        working-directory: ${{ steps.strings.outputs.artifact-dir }}
+        run: |
+          extra_args=""
+          if [[ "true" == "true" ]]; then
+            extra_args="--multithreaded "
+          fi
+          targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//')
+          echo "targets from input: ${targets_str}"
+          ${{ github.workspace }}/proof-producer/tests/make_proof_for_pairs.sh --use-nix ${extra_args} ${targets_str}
+
+      - name: Download transpiler output artifact
+        id: download-to-artifact
+        uses: dawidd6/action-download-artifact@v3
+        with:
+          repo: NilFoundation/zkLLVM
+          name: ${{ env.TO_ARTIFACT_NAME }}
+          path: ${{ steps.strings.outputs.artifact-dir }}
+          run_id: ${{ needs.get-zkllvm-run.outputs.run-id }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Merge proofs into transpiler output
+        working-directory: ${{ steps.strings.outputs.artifact-dir }}
+        run: |
+          copy_failed=0
+          while read dir; do
+            base_name=${dir#./transpiler_output_}
+
+            if [[ -d "$base_name" ]]; then
+              if ! cp "${base_name}/proof.bin" "${dir}/"; then
+                # fails if artifact contains dirs targets we didn't produce proofs for
+                echo "Failed to copy proof.bin to ${dir}" >&2
+                copy_failed=1
+              else
+                echo "proof.bin added to ${dir}"
+              fi
+            else
+              echo "Error: No matching directory found for ${dir}" >&2
+            fi
+          done < <(find . -type d -name "transpiler_output_*")
+
+          if [ $copy_failed -eq 1 ]; then
+            echo "One or more copy operations failed." >&2
+          fi
+
+      - name: Set aritfact name
+        id: artifact-name
+        run: |
+          echo "merged=transpiler-output-merged-proofs" >> $GITHUB_OUTPUT
+
+      - name: Upload merged artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.artifact-name.outputs.merged }}
+          path: |
+            ${{ steps.strings.outputs.artifact-dir }}/transpiler_output_*
+
+      # TODO: add package derivation to nix, upload its result
+      # - name: Upload .deb package
+      #   if: matrix.cpp-compiler == 'clang++' && matrix.build-type == 'Release'
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: proof-producer.deb
+      #     path: |
+      #        ${{ steps.strings.outputs.build-dir }}/proof-producer*.deb
+
+
+  verify-proof-producer-proofs:
+    name: Verify proof-producer proofs with EVM-placeholder
+    needs:
+      - build-and-generate-proofs
+      - prepare-targets
+    if: |
+      always() && !cancelled() &&
+      (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') &&
+      (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped')
+    uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
+    with:
+      artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }}
+      evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
+      test-names: ${{ needs.prepare-targets.outputs.evm-targets }}

.github/workflows/pull-request.yml

@@ -29,3 +29,10 @@ jobs:
       always() && !cancelled()
     secrets: inherit
 
+  check-proof-producer:
+    name: Check Proof Producer
+    uses: ./.github/workflows/check-proof-producer.yml
+    if: |
+      always() && !cancelled()
+    secrets: inherit
+

flake.nix

@@ -215,5 +215,15 @@
           };
           default = all-gcc;
         };
+        apps = {
+          single-threaded = {
+            type = "app";
+            program = "${self.packages.${system}.proof-producer}/bin/proof-producer-single-threaded";
+          };
+          default = {
+            type = "app";
+            program = "${self.packages.${system}.default}/bin/proof-producer-multi-threaded";
+          };
+        };
       }));
 }

proof-producer/CMakeLists.txt

@@ -19,7 +19,7 @@ cmake_policy(SET CMP0042 NEW)
 cmake_policy(SET CMP0028 NEW)
 cmake_policy(SET CMP0057 NEW)
 
-find_package(CM)
+find_package(CM REQUIRED)
 include(CMConfig)
 include(CMDeploy)
 include(CMSetupVersion)

proof-producer/tests/make_proof_for_pairs.sh

@@ -28,7 +28,7 @@ make_proof_for_pair() {
 
     if [ -f "$crct_file" ]; then
         mkdir -p "$proof_dir"  # Ensure the output directory exists
-        echo -n "Processing $tbl_file and $crct_file (proof will be at $proof_dir): "
+        echo -n "Processing $tbl_file and $crct_file (proof will be at $proof_dir; binary name: $proof_generator_binary): "
         if $proof_generator_binary -t "$tbl_file" --circuit "$crct_file" --proof "$proof_dir/proof.bin" ${args_to_forward[@]}; then
             color_green "success"
         else