Skip to content

Commit

Permalink
Merge pull request #8 from NilFoundation/directly-include-proof-producer
Browse files Browse the repository at this point in the history
Directly include proof producer
  • Loading branch information
AndreyMlashkin authored Sep 13, 2024
2 parents a75ba4d + b3b1772 commit b1862d2
Show file tree
Hide file tree
Showing 19 changed files with 1,848 additions and 6 deletions.
243 changes: 243 additions & 0 deletions .github/workflows/check-proof-producer.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
name: Check Proof producer

on:
workflow_call:

env:
CAA_ARTIFACT_NAME: circuits-and-assignments
TO_ARTIFACT_NAME: transpiler-output
INTEGRATION_TESTING_TARGETS: |
arithmetics_cpp_example
polynomial_cpp_example
poseidon_cpp_example
merkle_tree_poseidon_cpp_example
uint_remainder_cpp
compare_eq_cpp
private_input_cpp
# uint_shift_left
# uint_bit_decomposition
# uint_bit_composition

jobs:
prepare-targets:
name: Prepare targets strings
runs-on: ubuntu-22.04
if: |
always() && !cancelled()
outputs:
evm-targets: ${{ steps.get-targets.outputs.evm-targets }}
prover-targets: ${{ steps.get-targets.outputs.prover-targets }}
steps:
- name: Set targets for integration testing
id: get-targets
run: |
targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {print "transpiler_output_" $1}')
echo "evm-targets<<EOF" >> $GITHUB_OUTPUT
echo "${targets_str}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
echo "prover-targets<<EOF" >> $GITHUB_OUTPUT
echo "${{ env.INTEGRATION_TESTING_TARGETS }}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
get-zkllvm-run:
name: Get zkLLVM run to use artifacts from
if: |
always() && !cancelled()
runs-on: ubuntu-22.04
outputs:
run-id: ${{ steps.get-run-id.outputs.run-id }}
steps:
- name: Get run ID of zkLLVM
id: get-run-id
env:
GH_TOKEN: ${{ github.token }}
run: |
zkllvm_ref="master"
zkllvm_repo="NilFoundation/zkLLVM"
while read -r line; do
echo "$line"
if [[ $line == "${zkllvm_repo}:"* ]]; then
zkllvm_ref=$(echo "$line" | cut -d ' ' -f 2)
break
fi
done <<< "$prs_refs"
if [[ $zkllvm_ref == refs/pull/* ]]; then
echo "Considering reference ${zkllvm_ref} a pr ref"
pr_number=${zkllvm_ref#refs/pull/}
pr_number=${pr_number%/merge}
sha=$(gh api repos/${zkllvm_repo}/pulls/$pr_number --jq '.head.sha')
elif [[ $zkllvm_ref == refs/tags/* ]]; then
echo "Considering reference ${zkllvm_ref} a tag"
tag=${zkllvm_ref#refs/tags/}
sha=$(gh api repos/${zkllvm_repo}/git/ref/tags/$tag --jq '.object.sha')
else
echo "Considering reference ${zkllvm_ref} a branch"
branch=${zkllvm_ref#refs/heads/}
# We can already fetch run_id here, but better fit to common approach with extra query by sha
sha=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?branch=${branch}&status=completed&per_page=1" \
--jq '.workflow_runs[0].head_sha')
fi
echo "Using head sha: ${sha}"
run_id=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?head_sha=${sha}&status=completed&per_page=1" \
--jq '.workflow_runs[0].id')
if [ -z "${run_id}" ]; then
echo no run ID fetched
exit 1
fi
echo "Run ID: ${run_id}"
for artifact in "${{ env.CAA_ARTIFACT_NAME }}" "${{ env.TO_ARTIFACT_NAME }}"; do
# Check if the artifact exists in the run
if ! gh run view ${run_id} --repo ${zkllvm_repo} | grep "$artifact"; then
echo "Artifact '$artifact' not found in run ${run_id}"
exit 1
fi
done
echo "run-id=${run_id}" >> $GITHUB_OUTPUT
build-and-generate-proofs:
name: Build prover, generate proofs for circuits
runs-on: [self-hosted, Linux, X64, aws_autoscaling]
needs:
- get-zkllvm-run
- prepare-targets
if: |
always() && !cancelled() &&
(needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') &&
(needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped')
outputs:
artifact-name: ${{ steps.artifact-name.outputs.merged }}

steps:
# https://github.com/actions/checkout/issues/1552
- name: Clean up after previous checkout
run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*;

- name: Checkout Proof Producer
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive

- name: Checkout submodules to specified refs
if: inputs.submodules-refs != ''
uses: NilFoundation/ci-cd/actions/[email protected]
with:
paths: |
${{ github.workspace }}/**
!${{ github.workspace }}/
!${{ github.workspace }}/**/.git/**
- name: Run checks
run: nix build -L .?#packages.x86_64-linux.proof-producer

- name: Set usefull strings
id: strings
run: |
echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" >> $GITHUB_OUTPUT
echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)"
- name: Download circuits and assignments artifact
uses: dawidd6/action-download-artifact@v3
with:
repo: NilFoundation/zkLLVM
name: ${{ env.CAA_ARTIFACT_NAME }}
path: ${{ steps.strings.outputs.artifact-dir }}
run_id: ${{ needs.get-zkllvm-run.outputs.run-id }}
github_token: ${{ secrets.GITHUB_TOKEN }}
skip_unpack: true # It can't unpack such large files (>2Gb for some circuits)

- name: Extract circuits and assignments artifact
working-directory: ${{ steps.strings.outputs.artifact-dir }}
run: |
unzip -o ${{ env.CAA_ARTIFACT_NAME }}.zip
- name: List artifacts
working-directory: ${{ steps.strings.outputs.artifact-dir }}
run: find .

- name: Make proofs for pairs
working-directory: ${{ steps.strings.outputs.artifact-dir }}
run: |
extra_args=""
if [[ "true" == "true" ]]; then
extra_args="--multithreaded "
fi
targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//')
echo "targets from input: ${targets_str}"
${{ github.workspace }}/proof-producer/tests/make_proof_for_pairs.sh --use-nix ${extra_args} ${targets_str}
- name: Download transpiler output artifact
id: download-to-artifact
uses: dawidd6/action-download-artifact@v3
with:
repo: NilFoundation/zkLLVM
name: ${{ env.TO_ARTIFACT_NAME }}
path: ${{ steps.strings.outputs.artifact-dir }}
run_id: ${{ needs.get-zkllvm-run.outputs.run-id }}
github_token: ${{ secrets.GITHUB_TOKEN }}

- name: Merge proofs into transpiler output
working-directory: ${{ steps.strings.outputs.artifact-dir }}
run: |
copy_failed=0
while read dir; do
base_name=${dir#./transpiler_output_}
if [[ -d "$base_name" ]]; then
if ! cp "${base_name}/proof.bin" "${dir}/"; then
# fails if artifact contains dirs targets we didn't produce proofs for
echo "Failed to copy proof.bin to ${dir}" >&2
copy_failed=1
else
echo "proof.bin added to ${dir}"
fi
else
echo "Error: No matching directory found for ${dir}" >&2
fi
done < <(find . -type d -name "transpiler_output_*")
if [ $copy_failed -eq 1 ]; then
echo "One or more copy operations failed." >&2
fi
- name: Set aritfact name
id: artifact-name
run: |
echo "merged=transpiler-output-merged-proofs" >> $GITHUB_OUTPUT
- name: Upload merged artifact
uses: actions/upload-artifact@v3
with:
name: ${{ steps.artifact-name.outputs.merged }}
path: |
${{ steps.strings.outputs.artifact-dir }}/transpiler_output_*
# TODO: add package derivation to nix, upload its result
# - name: Upload .deb package
# if: matrix.cpp-compiler == 'clang++' && matrix.build-type == 'Release'
# uses: actions/upload-artifact@v4
# with:
# name: proof-producer.deb
# path: |
# ${{ steps.strings.outputs.build-dir }}/proof-producer*.deb


verify-proof-producer-proofs:
name: Verify proof-producer proofs with EVM-placeholder
needs:
- build-and-generate-proofs
- prepare-targets
if: |
always() && !cancelled() &&
(needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') &&
(needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped')
uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
with:
artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }}
evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
test-names: ${{ needs.prepare-targets.outputs.evm-targets }}
7 changes: 7 additions & 0 deletions .github/workflows/pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,10 @@ jobs:
always() && !cancelled()
secrets: inherit

check-proof-producer:
name: Check Proof Producer
uses: ./.github/workflows/check-proof-producer.yml
if: |
always() && !cancelled()
secrets: inherit

54 changes: 51 additions & 3 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,35 @@
crypto3 = crypto3;
});

proof-producer = (pkgs.callPackage ./proof-producer/proof-producer.nix {
runTests = false;
enableDebug = false;
crypto3 = crypto3;
transpiler = transpiler;
parallel-crypto3 = parallel-crypto3;
});
proof-producer-tests = (pkgs.callPackage ./proof-producer/proof-producer.nix {
runTests = true;
enableDebug = false;
crypto3 = crypto3;
transpiler = transpiler;
parallel-crypto3 = parallel-crypto3;
});
proof-producer-debug-tests = (pkgs.callPackage ./proof-producer/proof-producer.nix {
enableDebug = true;
runTests = true;
crypto3 = crypto3;
transpiler = transpiler;
parallel-crypto3 = parallel-crypto3;
});

# The "all" package will build all packages. Convenient for CI,
# so that "nix build" will check that all packages are correct.
# The packages that have no changes will not be rebuilt, and instead
# fetched from the cache.
all = pkgs.symlinkJoin {
name = "all";
paths = [ crypto3 evm-assigner zkevm-framework transpiler];
paths = [ crypto3 evm-assigner zkevm-framework transpiler proof-producer];
};
default = all;
};
Expand Down Expand Up @@ -167,15 +189,41 @@
crypto3 = crypto3-clang;
});

proof-producer-gcc = (pkgs.callPackage ./proof-producer/proof-producer.nix {
runTests = true;
enableDebug = false;
crypto3 = packages.crypto3;
transpiler = packages.transpiler;
parallel-crypto3 = packages.parallel-crypto3;
});
proof-producer-clang = (pkgs.callPackage ./proof-producer/proof-producer.nix {
stdenv = pkgs.llvmPackages_18.stdenv;
runTests = true;
enableDebug = false;
crypto3 = crypto3-clang;
transpiler = transpiler-clang;
parallel-crypto3 = parallel-crypto3-clang;
});

all-clang = pkgs.symlinkJoin {
name = "all";
paths = [ crypto3-clang parallel-crypto3-clang evm-assigner-clang transpiler-clang ];
paths = [ crypto3-clang parallel-crypto3-clang evm-assigner-clang transpiler-clang proof-producer-clang ];
};
all-gcc = pkgs.symlinkJoin {
name = "all";
paths = [ crypto3-gcc parallel-crypto3-gcc evm-assigner-gcc zkevm-framework-gcc transpiler-gcc ];
paths = [ crypto3-gcc parallel-crypto3-gcc evm-assigner-gcc zkevm-framework-gcc transpiler-gcc proof-producer-gcc ];
};
default = all-gcc;
};
apps = {
single-threaded = {
type = "app";
program = "${self.packages.${system}.proof-producer}/bin/proof-producer-single-threaded";
};
default = {
type = "app";
program = "${self.packages.${system}.default}/bin/proof-producer-multi-threaded";
};
};
}));
}
12 changes: 12 additions & 0 deletions proof-producer/.pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace

- repo: https://github.com/pre-commit/mirrors-clang-format
rev: 'v18.1.6' # Use the sha / tag you want to point at
hooks:
- id: clang-format
Loading

0 comments on commit b1862d2

Please sign in to comment.