-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #8 from NilFoundation/directly-include-proof-producer
Directly include proof producer
- Loading branch information
Showing
19 changed files
with
1,848 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,243 @@ | ||
| name: Check Proof producer | ||
|
|
||
| on: | ||
| workflow_call: | ||
|
|
||
| env: | ||
| CAA_ARTIFACT_NAME: circuits-and-assignments | ||
| TO_ARTIFACT_NAME: transpiler-output | ||
| INTEGRATION_TESTING_TARGETS: | | ||
| arithmetics_cpp_example | ||
| polynomial_cpp_example | ||
| poseidon_cpp_example | ||
| merkle_tree_poseidon_cpp_example | ||
| uint_remainder_cpp | ||
| compare_eq_cpp | ||
| private_input_cpp | ||
| # uint_shift_left | ||
| # uint_bit_decomposition | ||
| # uint_bit_composition | ||
|
|
||
| jobs: | ||
| prepare-targets: | ||
| name: Prepare targets strings | ||
| runs-on: ubuntu-22.04 | ||
| if: | | ||
| always() && !cancelled() | ||
| outputs: | ||
| evm-targets: ${{ steps.get-targets.outputs.evm-targets }} | ||
| prover-targets: ${{ steps.get-targets.outputs.prover-targets }} | ||
| steps: | ||
| - name: Set targets for integration testing | ||
| id: get-targets | ||
| run: | | ||
| targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {print "transpiler_output_" $1}') | ||
| echo "evm-targets<<EOF" >> $GITHUB_OUTPUT | ||
| echo "${targets_str}" >> $GITHUB_OUTPUT | ||
| echo "EOF" >> $GITHUB_OUTPUT | ||
| echo "prover-targets<<EOF" >> $GITHUB_OUTPUT | ||
| echo "${{ env.INTEGRATION_TESTING_TARGETS }}" >> $GITHUB_OUTPUT | ||
| echo "EOF" >> $GITHUB_OUTPUT | ||
| get-zkllvm-run: | ||
| name: Get zkLLVM run to use artifacts from | ||
| if: | | ||
| always() && !cancelled() | ||
| runs-on: ubuntu-22.04 | ||
| outputs: | ||
| run-id: ${{ steps.get-run-id.outputs.run-id }} | ||
| steps: | ||
| - name: Get run ID of zkLLVM | ||
| id: get-run-id | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| run: | | ||
| zkllvm_ref="master" | ||
| zkllvm_repo="NilFoundation/zkLLVM" | ||
| while read -r line; do | ||
| echo "$line" | ||
| if [[ $line == "${zkllvm_repo}:"* ]]; then | ||
| zkllvm_ref=$(echo "$line" | cut -d ' ' -f 2) | ||
| break | ||
| fi | ||
| done <<< "$prs_refs" | ||
| if [[ $zkllvm_ref == refs/pull/* ]]; then | ||
| echo "Considering reference ${zkllvm_ref} a pr ref" | ||
| pr_number=${zkllvm_ref#refs/pull/} | ||
| pr_number=${pr_number%/merge} | ||
| sha=$(gh api repos/${zkllvm_repo}/pulls/$pr_number --jq '.head.sha') | ||
| elif [[ $zkllvm_ref == refs/tags/* ]]; then | ||
| echo "Considering reference ${zkllvm_ref} a tag" | ||
| tag=${zkllvm_ref#refs/tags/} | ||
| sha=$(gh api repos/${zkllvm_repo}/git/ref/tags/$tag --jq '.object.sha') | ||
| else | ||
| echo "Considering reference ${zkllvm_ref} a branch" | ||
| branch=${zkllvm_ref#refs/heads/} | ||
| # We can already fetch run_id here, but better fit to common approach with extra query by sha | ||
| sha=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?branch=${branch}&status=completed&per_page=1" \ | ||
| --jq '.workflow_runs[0].head_sha') | ||
| fi | ||
| echo "Using head sha: ${sha}" | ||
| run_id=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?head_sha=${sha}&status=completed&per_page=1" \ | ||
| --jq '.workflow_runs[0].id') | ||
| if [ -z "${run_id}" ]; then | ||
| echo no run ID fetched | ||
| exit 1 | ||
| fi | ||
| echo "Run ID: ${run_id}" | ||
| for artifact in "${{ env.CAA_ARTIFACT_NAME }}" "${{ env.TO_ARTIFACT_NAME }}"; do | ||
| # Check if the artifact exists in the run | ||
| if ! gh run view ${run_id} --repo ${zkllvm_repo} | grep "$artifact"; then | ||
| echo "Artifact '$artifact' not found in run ${run_id}" | ||
| exit 1 | ||
| fi | ||
| done | ||
| echo "run-id=${run_id}" >> $GITHUB_OUTPUT | ||
| build-and-generate-proofs: | ||
| name: Build prover, generate proofs for circuits | ||
| runs-on: [self-hosted, Linux, X64, aws_autoscaling] | ||
| needs: | ||
| - get-zkllvm-run | ||
| - prepare-targets | ||
| if: | | ||
| always() && !cancelled() && | ||
| (needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') && | ||
| (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') | ||
| outputs: | ||
| artifact-name: ${{ steps.artifact-name.outputs.merged }} | ||
|
|
||
| steps: | ||
| # https://github.com/actions/checkout/issues/1552 | ||
| - name: Clean up after previous checkout | ||
| run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*; | ||
|
|
||
| - name: Checkout Proof Producer | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
| submodules: recursive | ||
|
|
||
| - name: Checkout submodules to specified refs | ||
| if: inputs.submodules-refs != '' | ||
| uses: NilFoundation/ci-cd/actions/[email protected] | ||
| with: | ||
| paths: | | ||
| ${{ github.workspace }}/** | ||
| !${{ github.workspace }}/ | ||
| !${{ github.workspace }}/**/.git/** | ||
| - name: Run checks | ||
| run: nix build -L .?#packages.x86_64-linux.proof-producer | ||
|
|
||
| - name: Set usefull strings | ||
| id: strings | ||
| run: | | ||
| echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" >> $GITHUB_OUTPUT | ||
| echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" | ||
| - name: Download circuits and assignments artifact | ||
| uses: dawidd6/action-download-artifact@v3 | ||
| with: | ||
| repo: NilFoundation/zkLLVM | ||
| name: ${{ env.CAA_ARTIFACT_NAME }} | ||
| path: ${{ steps.strings.outputs.artifact-dir }} | ||
| run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| skip_unpack: true # It can't unpack such large files (>2Gb for some circuits) | ||
|
|
||
| - name: Extract circuits and assignments artifact | ||
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | ||
| run: | | ||
| unzip -o ${{ env.CAA_ARTIFACT_NAME }}.zip | ||
| - name: List artifacts | ||
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | ||
| run: find . | ||
|
|
||
| - name: Make proofs for pairs | ||
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | ||
| run: | | ||
| extra_args="" | ||
| if [[ "true" == "true" ]]; then | ||
| extra_args="--multithreaded " | ||
| fi | ||
| targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//') | ||
| echo "targets from input: ${targets_str}" | ||
| ${{ github.workspace }}/proof-producer/tests/make_proof_for_pairs.sh --use-nix ${extra_args} ${targets_str} | ||
| - name: Download transpiler output artifact | ||
| id: download-to-artifact | ||
| uses: dawidd6/action-download-artifact@v3 | ||
| with: | ||
| repo: NilFoundation/zkLLVM | ||
| name: ${{ env.TO_ARTIFACT_NAME }} | ||
| path: ${{ steps.strings.outputs.artifact-dir }} | ||
| run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Merge proofs into transpiler output | ||
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | ||
| run: | | ||
| copy_failed=0 | ||
| while read dir; do | ||
| base_name=${dir#./transpiler_output_} | ||
| if [[ -d "$base_name" ]]; then | ||
| if ! cp "${base_name}/proof.bin" "${dir}/"; then | ||
| # fails if artifact contains dirs targets we didn't produce proofs for | ||
| echo "Failed to copy proof.bin to ${dir}" >&2 | ||
| copy_failed=1 | ||
| else | ||
| echo "proof.bin added to ${dir}" | ||
| fi | ||
| else | ||
| echo "Error: No matching directory found for ${dir}" >&2 | ||
| fi | ||
| done < <(find . -type d -name "transpiler_output_*") | ||
| if [ $copy_failed -eq 1 ]; then | ||
| echo "One or more copy operations failed." >&2 | ||
| fi | ||
| - name: Set aritfact name | ||
| id: artifact-name | ||
| run: | | ||
| echo "merged=transpiler-output-merged-proofs" >> $GITHUB_OUTPUT | ||
| - name: Upload merged artifact | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: ${{ steps.artifact-name.outputs.merged }} | ||
| path: | | ||
| ${{ steps.strings.outputs.artifact-dir }}/transpiler_output_* | ||
| # TODO: add package derivation to nix, upload its result | ||
| # - name: Upload .deb package | ||
| # if: matrix.cpp-compiler == 'clang++' && matrix.build-type == 'Release' | ||
| # uses: actions/upload-artifact@v4 | ||
| # with: | ||
| # name: proof-producer.deb | ||
| # path: | | ||
| # ${{ steps.strings.outputs.build-dir }}/proof-producer*.deb | ||
|
|
||
|
|
||
| verify-proof-producer-proofs: | ||
| name: Verify proof-producer proofs with EVM-placeholder | ||
| needs: | ||
| - build-and-generate-proofs | ||
| - prepare-targets | ||
| if: | | ||
| always() && !cancelled() && | ||
| (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') && | ||
| (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') | ||
| uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d | ||
| with: | ||
| artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }} | ||
| evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d | ||
| test-names: ${{ needs.prepare-targets.outputs.evm-targets }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| repos: | ||
| - repo: https://github.com/pre-commit/pre-commit-hooks | ||
| rev: v2.3.0 | ||
| hooks: | ||
| - id: check-yaml | ||
| - id: end-of-file-fixer | ||
| - id: trailing-whitespace | ||
|
|
||
| - repo: https://github.com/pre-commit/mirrors-clang-format | ||
| rev: 'v18.1.6' # Use the sha / tag you want to point at | ||
| hooks: | ||
| - id: clang-format |
Oops, something went wrong.