From 6255f0091afb99f92233350bad1a59f542103942 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 15 Jul 2024 16:49:51 +0800 Subject: [PATCH] Reviewable. Automated testing missing. --- .cargo/config.toml | 2 +- Cargo.lock | 28 +++ Cargo.toml | 29 ++- evm_arithmetization/Cargo.toml | 7 +- .../benches/fibonacci_25m_gas.rs | 1 + .../src/cpu/kernel/interpreter.rs | 87 +++++++- .../src/cpu/kernel/tests/add11.rs | 2 + .../kernel/tests/core/jumpdest_analysis.rs | 10 + evm_arithmetization/src/generation/mod.rs | 8 +- .../src/generation/prover_input.rs | 102 +++++++++- evm_arithmetization/src/generation/state.rs | 5 +- evm_arithmetization/src/lib.rs | 2 + evm_arithmetization/src/witness/operation.rs | 1 + evm_arithmetization/tests/add11_yml.rs | 1 + .../tests/basic_smart_contract.rs | 1 + evm_arithmetization/tests/empty_txn_list.rs | 1 + evm_arithmetization/tests/erc20.rs | 1 + evm_arithmetization/tests/erc721.rs | 1 + evm_arithmetization/tests/log_opcode.rs | 4 + .../tests/self_balance_gas_cost.rs | 1 + evm_arithmetization/tests/selfdestruct.rs | 1 + evm_arithmetization/tests/simple_transfer.rs | 1 + evm_arithmetization/tests/two_to_one_block.rs | 1 + evm_arithmetization/tests/withdrawals.rs | 1 + trace_decoder/src/decoding.rs | 2 + trace_decoder/src/lib.rs | 14 +- trace_decoder/src/processed_block_trace.rs | 3 + zero_bin/common/Cargo.toml | 1 + zero_bin/rpc/Cargo.toml | 2 + zero_bin/rpc/src/native/mod.rs | 4 - zero_bin/rpc/src/native/txn.rs | 185 ++++++++++++++++-- zero_bin/tools/prove_stdio.sh | 3 +- 32 files changed, 472 insertions(+), 40 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6340ce34a..a71f56369 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,4 @@ [build] # https://github.com/rust-lang/rust/pull/124129 # https://github.com/dtolnay/linkme/pull/88 -rustflags = ["-Z", "linker-features=-lld"] +rustflags = ["-C", "target-cpu=native", "-Z", "linker-features=-lld"] diff --git a/Cargo.lock b/Cargo.lock index 0302d8d8d..37557b10f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1674,6 +1674,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.9.0" @@ -1905,6 +1911,7 @@ dependencies = [ name = "evm_arithmetization" version = "0.2.0" dependencies = [ + "alloy", "anyhow", "bytes", "criterion", @@ -1925,6 +1932,8 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon", "plonky2_util", + "pretty_assertions", + "primitive-types 0.12.2", "rand", "rand_chacha", "ripemd", @@ -3611,6 +3620,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "pretty_env_logger" version = "0.5.0" @@ -4002,9 +4021,11 @@ dependencies = [ "anyhow", "clap", "compat", + "env_logger 0.11.3", "evm_arithmetization", "futures", "hex", + "log", "lru", "mpt_trie", "primitive-types 0.12.2", @@ -5581,6 +5602,12 @@ dependencies = [ "time", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zero_bin_common" version = "0.1.0" @@ -5592,6 +5619,7 @@ dependencies = [ "evm_arithmetization", "futures", "plonky2", + "pretty_assertions", "proof_gen", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 3d9a44604..e0b75d74c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,6 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" -enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -94,7 +93,6 @@ ruint = "1.12.3" serde = "1.0.203" serde_json = "1.0.118" serde_path_to_error = "0.1.16" -serde_with = "3.8.1" sha2 = "0.10.8" smt_trie = { path = "smt_trie", version = "0.1.0" } static_assertions = "1.1.0" @@ -110,6 +108,8 @@ u4 = "0.1.0" uint = "0.9.5" url = "2.5.2" winnow = "0.6.13" +evm-disassembler = "0.5.0" +pretty_assertions = "1.4.0" # zero-bin related dependencies ops = { path = "zero_bin/ops" } @@ -128,4 +128,29 @@ proc-macro2 = "1.0" quote = "1.0" syn = "2.0" trybuild = "1.0" + zk_evm_proc_macro = { path = "proc_macro" } + +[profile.release] +opt-level=3 +debug=true +incremental=true +debug-assertions=true +lto=false +overflow-checks=false + +[profile.test] +opt-level=3 +debug=true +incremental=true +debug-assertions=true +lto=false +overflow-checks=false + +[profile.dev] +opt-level=3 +debug=true +incremental=true +debug-assertions=true +lto=false +overflow-checks=false diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index 8d98c214b..97fd323d9 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -15,11 +15,12 @@ homepage.workspace = true keywords.workspace = true [dependencies] +__compat_primitive_types = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } ethereum-types = { workspace = true } -hex = { workspace = true, optional = true } +hex = { workspace = true } hex-literal = { workspace = true } itertools = { workspace = true } keccak-hash = { workspace = true } @@ -46,6 +47,8 @@ serde_json = { workspace = true } # Local dependencies mpt_trie = { workspace = true } zk_evm_proc_macro = { workspace = true } +pretty_assertions = { workspace = true } +alloy.workspace = true [dev-dependencies] criterion = { workspace = true } @@ -55,7 +58,7 @@ sha2 = { workspace = true } [features] default = ["parallel"] -asmtools = ["hex"] +#asmtools = ["hex"] parallel = [ "plonky2/parallel", "plonky2_maybe_rayon/parallel", diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index d27b7e5ce..e49ceed57 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -173,6 +173,7 @@ fn prepare_setup() -> anyhow::Result { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index e47d48e92..e45604c04 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -8,18 +8,24 @@ use core::cmp::Ordering; use std::collections::{BTreeSet, HashMap}; +use alloy::primitives::Address; use anyhow::anyhow; -use ethereum_types::{BigEndianHash, U256}; +use ethereum_types::{BigEndianHash, H256, U256}; use log::Level; use mpt_trie::partial_trie::PartialTrie; use plonky2::field::types::Field; +//use alloy::primitives::Address; use crate::byte_packing::byte_packing_stark::BytePackingOp; use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; use crate::generation::mpt::load_all_mpts; +use crate::generation::prover_input::{ + get_proofs_and_jumpdests, CodeDb, ContextJumpDests, JumpDestTableProcessed, + JumpDestTableWitness, +}; use crate::generation::rlp::all_rlp_prover_inputs_reversed; use crate::generation::state::{ all_withdrawals_prover_inputs_reversed, GenerationState, GenerationStateCheckpoint, @@ -90,6 +96,81 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( } } +// todo: remove. for easy comparison only +pub(crate) fn simulate_cpu_and_get_user_jumps_rpc( + final_label: &str, + state: &GenerationState, +) -> Option { + match state.jumpdest_table { + Some(_) => None, + None => { + let halt_pc = KERNEL.global_labels[final_label]; + let initial_context = state.registers.context; + let mut interpreter = + Interpreter::new_with_state_and_halt_condition(state, halt_pc, initial_context); + + log::debug!("Simulating CPU for jumpdest analysis."); + + let _ = interpreter.run(); + + log::trace!("jumpdest table = {:?}", interpreter.jumpdest_table); + + //debug_assert_eq!(&interpreter.jumpdest_table, &state.inputs.jumpdest_table); + let jumpdest_table = set_jumpdest_analysis_inputs_rpc( + &state.inputs.jumpdest_table, + &state.inputs.contract_code, + ); + + Some(jumpdest_table) + } + } +} + +/// Computes the proofs for each context. +/// +/// # Arguments +/// +/// - `jumpdest_table_rpc`: The raw table received from RPC. +/// - `code_db`: The corresponding database of contract code used in the trace. +pub(crate) fn set_jumpdest_analysis_inputs_rpc( + jumpdest_table_rpc: &JumpDestTableWitness, + code_db: &CodeDb, +) -> JumpDestTableProcessed { + let ctx_proofs = jumpdest_table_rpc + .0 + .iter() + .flat_map(|(code_addr, ctx_jumpdests)| { + prove_context_jumpdests(&code_db[code_addr], ctx_jumpdests) + }) + .collect(); + JumpDestTableProcessed(ctx_proofs) +} + +/// Orchestrates the proving of all contexts in a specific bytecode. +/// +/// # Arguments +/// +/// - `ctx_jumpdests`: Map from `ctx` to its list of offsets to reached +/// `JUMPDEST`s. +/// - `code`: The bytecode for the contexts. This is the same for all contexts. +fn prove_context_jumpdests( + code: &[u8], + ctx_jumpdests: &ContextJumpDests, +) -> HashMap> { + ctx_jumpdests + .0 + .iter() + .map(|(&ctx, jumpdests)| { + let proofs = jumpdests + .last() + .map_or(Vec::default(), |&largest_address| { + get_proofs_and_jumpdests(code, largest_address, jumpdests.clone()) + }); + (ctx, proofs) + }) + .collect() +} + impl Interpreter { /// Returns an instance of `Interpreter` given `GenerationInputs`, and /// assuming we are initializing with the `KERNEL` code. @@ -350,6 +431,8 @@ impl Interpreter { } pub(crate) fn add_jumpdest_offset(&mut self, offset: usize) { + // println!("SIM: ({:?}, {:?})", &self.generation_state.registers.context, + // offset); if let Some(jumpdest_table) = self .jumpdest_table .get_mut(&self.generation_state.registers.context) @@ -506,7 +589,7 @@ impl State for Interpreter { let registers = self.generation_state.registers; let (mut row, opcode) = self.base_row(); - let op = decode(registers, opcode)?; + let op: Operation = decode(registers, opcode)?; fill_op_flag(op, &mut row); diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index c2be6a0bd..511bfbca1 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -152,6 +152,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let initial_stack = vec![]; @@ -293,6 +294,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index dd9a295e2..de578d707 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -210,3 +210,13 @@ fn test_verify_non_jumpdest() -> Result<()> { } Ok(()) } + +#[test] +fn compare_jumpdest_tables() -> Result<()> { + let jumpdest_tbl_sim = todo!(); + let jumpdest_tbl_rpc = todo!(); + + assert_eq!(jumpdest_tbl_sim, jumpdest_tbl_rpc); + + Ok(()) +} \ No newline at end of file diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 84da8ceb2..7651b5bc8 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -1,7 +1,8 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{Address, BigEndianHash, H256, U256}; +// use hex::ToHex; use log::log_enabled; use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; use plonky2::field::extension::Extendable; @@ -10,6 +11,7 @@ use plonky2::field::types::Field; use plonky2::hash::hash_types::RichField; use plonky2::timed; use plonky2::util::timing::TimingTree; +use prover_input::JumpDestTableWitness; use serde::{Deserialize, Serialize}; use starky::config::StarkConfig; use GlobalMetadata::{ @@ -76,6 +78,9 @@ pub struct GenerationInputs { /// The hash of the current block, and a list of the 256 previous block /// hashes. pub block_hashes: BlockHashes, + + /// A jumptable describing each JUMPDEST reached(jumped to?). + pub jumpdest_table: JumpDestTableWitness, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -223,6 +228,7 @@ pub fn generate_traces, const D: usize>( timing: &mut TimingTree, ) -> anyhow::Result<([Vec>; NUM_TABLES], PublicValues)> { debug_inputs(&inputs); + // TODO let mut state = GenerationState::::new(inputs.clone(), &KERNEL.code) .map_err(|err| anyhow!("Failed to parse all the initial prover inputs: {:?}", err))?; diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 904524be2..6d4821589 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -1,17 +1,23 @@ use core::mem::transmute; use std::collections::{BTreeSet, HashMap}; +use std::fmt::Display; use std::str::FromStr; +use alloy::primitives::Address; use anyhow::{bail, Error}; use ethereum_types::{BigEndianHash, H256, U256, U512}; +use hex::{FromHex, ToHex}; use itertools::Itertools; use num_bigint::BigUint; use plonky2::field::types::Field; +use pretty_assertions::assert_eq; use serde::{Deserialize, Serialize}; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; -use crate::cpu::kernel::interpreter::simulate_cpu_and_get_user_jumps; +use crate::cpu::kernel::interpreter::{ + set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, +}; use crate::extension_tower::{FieldExt, Fp12, BLS381, BN254}; use crate::generation::prover_input::EvmField::{ Bls381Base, Bls381Scalar, Bn254Base, Bn254Scalar, Secp256k1Base, Secp256k1Scalar, @@ -27,6 +33,47 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; +pub type CodeDb = HashMap<__compat_primitive_types::H256, Vec>; + +/// Each `CodeAddress` can be called one or more times, each time creating a new +/// `Context`. Each `Context` may will have zero or more `JumpDests`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct ContextJumpDests(pub HashMap>); + +/// Map from `CodeAddress -> (Context -> [JumpDests])` +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct JumpDestTableWitness(pub HashMap); + +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct JumpDestTableProcessed(pub HashMap>); + +impl Display for JumpDestTableWitness { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "=== JumpDest table ===\n"); + + for (code, ctxtbls) in &self.0 { + write!(f, "codehash: {:?}\n{}", code, ctxtbls)?; + // for ct in &ctxtbls.0 { + // write!(f, "{}\n", ct)?; + // } + } + Ok(()) + } +} + +impl Display for ContextJumpDests { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for (ctx, offsets) in &self.0 { + write!(f, " ctx: {}, offsets: [", ctx)?; + for offset in offsets { + write!(f, "{:#10x} ", offset)?; + } + write!(f, "]\n")?; + } + Ok(()) + } +} + /// Prover input function represented as a scoped function name. /// Example: `PROVER_INPUT(ff::bn254_base::inverse)` is represented as /// `ProverInputFn([ff, bn254_base, inverse])`. @@ -266,6 +313,7 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // This should take care of itself when the table is already filled in.. if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } @@ -386,7 +434,54 @@ impl GenerationState { fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - self.jumpdest_table = simulate_cpu_and_get_user_jumps("terminate_common", self); + + // TODO: This function will be pruned for everything not relating to + // `jumpdest_table_rpc`. Leaving it as is until test structure is in + // place. This is essentially the test property, we want to check: + // assert_eq!(&jumpdest_table_sim, &jumpdest_table_rpc); + let jumpdest_table_sim = simulate_cpu_and_get_user_jumps("terminate_common", self); + log::debug!("SIM JUMPDEST table"); + log::debug!( + "{:?}", + &jumpdest_table_sim.as_ref().unwrap().keys().sorted() + ); + log::debug!("{}", &jumpdest_table_sim.as_ref().unwrap().keys().len()); + + let jumpdest_table_rpc = { + let jumpdest_table = set_jumpdest_analysis_inputs_rpc( + &self.inputs.jumpdest_table, + &self.inputs.contract_code, + ); + Some(jumpdest_table.0) + }; + log::debug!("RPC JUMPDEST table"); + log::debug!( + "{:?}", + &jumpdest_table_rpc.as_ref().unwrap().keys().sorted() + ); + log::debug!("{}", &jumpdest_table_rpc.as_ref().unwrap().keys().len()); + + assert_eq!( + &jumpdest_table_sim.as_ref().unwrap().keys().len(), + &jumpdest_table_rpc.as_ref().unwrap().keys().len() + ); + assert_eq!( + jumpdest_table_sim + .as_ref() + .unwrap() + .keys() + .sorted() + .collect::>(), + jumpdest_table_rpc + .as_ref() + .unwrap() + .keys() + .sorted() + .collect::>() + ); + assert_eq!(&jumpdest_table_sim, &jumpdest_table_rpc); + + self.jumpdest_table = jumpdest_table_rpc; Ok(()) } @@ -495,7 +590,7 @@ impl GenerationState { /// for which none of the previous 32 bytes in the code (including opcodes /// and pushed bytes) is a PUSHXX and the address is in its range. It returns /// a vector of even size containing proofs followed by their addresses. -fn get_proofs_and_jumpdests( +pub(crate) fn get_proofs_and_jumpdests( code: &[u8], largest_address: usize, jumpdest_table: std::collections::BTreeSet, @@ -515,6 +610,7 @@ fn get_proofs_and_jumpdests( false }; let last_proof = if has_prefix { addr - 32 } else { last_proof }; + // todo if jumpdest_table.contains(&addr) { // Push the proof proofs.push(last_proof); diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index 6e08c5391..30da89ffb 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::mem::size_of; use anyhow::{anyhow, bail}; @@ -302,6 +302,8 @@ pub(crate) struct GenerationState { /// Pointers, within the `TrieData` segment, of the three MPTs. pub(crate) trie_root_ptrs: TrieRootPtrs, + //pub(crate) jumpdest_table_rpc: HashMap>, + /// A hash map where the key is a context in the user's code and the value /// is the set of jump destinations with its corresponding "proof". A /// "proof" for a jump destination is either 0 or an address i > 32 in @@ -340,6 +342,7 @@ impl GenerationState { txn_root_ptr: 0, receipt_root_ptr: 0, }, + // TODO jumpdest_table: None, }; let trie_root_ptrs = state.preinitialize_mpts(&inputs.tries); diff --git a/evm_arithmetization/src/lib.rs b/evm_arithmetization/src/lib.rs index b9236ae96..662f222d1 100644 --- a/evm_arithmetization/src/lib.rs +++ b/evm_arithmetization/src/lib.rs @@ -206,6 +206,8 @@ pub mod verifier; pub mod generation; pub mod witness; +pub use generation::prover_input::{JumpDestTableProcessed, JumpDestTableWitness, CodeDb}; + // Utility modules pub mod curve_pairings; pub mod extension_tower; diff --git a/evm_arithmetization/src/witness/operation.rs b/evm_arithmetization/src/witness/operation.rs index 07d48819b..c5a6702b9 100644 --- a/evm_arithmetization/src/witness/operation.rs +++ b/evm_arithmetization/src/witness/operation.rs @@ -286,6 +286,7 @@ pub(crate) fn generate_set_context>( let old_ctx = generation_state.registers.context; // The popped value needs to be scaled down. let new_ctx = u256_to_usize(ctx >> CONTEXT_SCALING_FACTOR)?; + eprintln!("ctx: {} -> {}", &old_ctx, &new_ctx); let sp_field = ContextMetadata::StackSize.unscale(); let old_sp_addr = MemoryAddress::new(old_ctx, Segment::ContextMetadata, sp_field); diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 00d7e56b4..75148ebb6 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -163,6 +163,7 @@ fn add11_yml() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/basic_smart_contract.rs b/evm_arithmetization/tests/basic_smart_contract.rs index fd0948d80..9a41c4402 100644 --- a/evm_arithmetization/tests/basic_smart_contract.rs +++ b/evm_arithmetization/tests/basic_smart_contract.rs @@ -195,6 +195,7 @@ fn test_basic_smart_contract() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/empty_txn_list.rs b/evm_arithmetization/tests/empty_txn_list.rs index 1205414f6..031b7232c 100644 --- a/evm_arithmetization/tests/empty_txn_list.rs +++ b/evm_arithmetization/tests/empty_txn_list.rs @@ -69,6 +69,7 @@ fn test_empty_txn_list() -> anyhow::Result<()> { prev_hashes: initial_block_hashes, cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; // Initialize the preprocessed circuits for the zkEVM. diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 609579af9..b6f83072f 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -171,6 +171,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 86dd34002..40cec1b3a 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -174,6 +174,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 75cdd44f5..129dd0120 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -233,6 +233,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); @@ -443,6 +444,7 @@ fn test_log_with_aggreg() -> anyhow::Result<()> { prev_hashes: block_hashes.clone(), cur_hash: block_1_hash, }, + jumpdest_table: Default::default(), }; // Preprocess all circuits. @@ -573,6 +575,7 @@ fn test_log_with_aggreg() -> anyhow::Result<()> { prev_hashes: block_hashes.clone(), cur_hash: block_1_hash, }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove root second", log::Level::Info); @@ -639,6 +642,7 @@ fn test_log_with_aggreg() -> anyhow::Result<()> { prev_hashes: block_hashes, cur_hash: block_2_hash, }, + jumpdest_table: Default::default(), }; let (root_proof, public_values) = diff --git a/evm_arithmetization/tests/self_balance_gas_cost.rs b/evm_arithmetization/tests/self_balance_gas_cost.rs index f4aa8a606..c7eb32a06 100644 --- a/evm_arithmetization/tests/self_balance_gas_cost.rs +++ b/evm_arithmetization/tests/self_balance_gas_cost.rs @@ -177,6 +177,7 @@ fn self_balance_gas_cost() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 0ef48d2f4..07a04370d 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -134,6 +134,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 8adbc1c42..812ede0c1 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -150,6 +150,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/evm_arithmetization/tests/two_to_one_block.rs b/evm_arithmetization/tests/two_to_one_block.rs index fe387cee2..aa3edf60a 100644 --- a/evm_arithmetization/tests/two_to_one_block.rs +++ b/evm_arithmetization/tests/two_to_one_block.rs @@ -194,6 +194,7 @@ fn get_test_block_proof( contract_code: Default::default(), block_metadata: inputs.block_metadata.clone(), block_hashes: inputs.block_hashes.clone(), + jumpdest_table: Default::default(), }; let timing = &mut TimingTree::new(&format!("Blockproof {timestamp}"), log::Level::Info); diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 7f133518b..5c63db393 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -80,6 +80,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_table: Default::default(), }; let mut timing = TimingTree::new("prove", log::Level::Debug); diff --git a/trace_decoder/src/decoding.rs b/trace_decoder/src/decoding.rs index 40a17c887..727a29bee 100644 --- a/trace_decoder/src/decoding.rs +++ b/trace_decoder/src/decoding.rs @@ -678,6 +678,7 @@ impl ProcessedBlockTrace { contract_code: txn_info.contract_code_accessed, block_metadata: other_data.b_data.b_meta.clone(), block_hashes: other_data.b_data.b_hashes.clone(), + jumpdest_table: txn_info.meta.jumpdest_table, }; // After processing a transaction, we update the remaining accumulators @@ -792,6 +793,7 @@ fn create_dummy_gen_input_common( gas_used_after: extra_data.gas_used_after, contract_code: HashMap::default(), withdrawals: vec![], // this is set after creating dummy payloads + jumpdest_table: Default::default(), } } diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index d5603012b..82dc96a91 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -35,7 +35,7 @@ //! if we can get this information sent to us instead. //! //! This library generates an Intermediary Representation (IR) of -//! a block's transactions, given a [BlockTrace] and some additional +//! a block's transactions, given a [`BlockTrace`] and some additional //! data represented by [OtherBlockData]. //! //! It first preprocesses the [BlockTrace] to provide transaction, @@ -97,7 +97,7 @@ use std::collections::HashMap; use ethereum_types::{Address, U256}; use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; -use evm_arithmetization::GenerationInputs; +use evm_arithmetization::{GenerationInputs, JumpDestTableWitness}; use keccak_hash::H256; use mpt_trie::partial_trie::HashedPartialTrie; use serde::{Deserialize, Serialize}; @@ -122,6 +122,7 @@ pub struct BlockTrace { /// Traces and other info per transaction. The index of the transaction /// within the block corresponds to the slot in this vec. pub txn_info: Vec, + } /// Minimal hashed out tries needed by all txns in the block. @@ -205,6 +206,10 @@ pub struct TxnMeta { /// Gas used by this txn (Note: not cumulative gas used). pub gas_used: u64, + + /// JumpDest table + //#[serde(with = "crate::hex")] + pub jumpdest_table: JumpDestTableWitness, } /// A "trace" specific to an account for a txn. @@ -258,7 +263,8 @@ pub enum ContractCodeUsage { } impl ContractCodeUsage { - fn get_code_hash(&self) -> H256 { + /// Get code hash from a read or write operation of contract code. + pub fn get_code_hash(&self) -> H256 { match self { ContractCodeUsage::Read(hash) => *hash, ContractCodeUsage::Write(bytes) => hash(bytes), @@ -402,6 +408,8 @@ pub fn entrypoint( }) .collect::>(); + //other.jumpdest_tables = Some(jumpdest_tables); + Ok(ProcessedBlockTrace { tries: pre_images.tries, txn_info, diff --git a/trace_decoder/src/processed_block_trace.rs b/trace_decoder/src/processed_block_trace.rs index 7c9e89757..576ce49cb 100644 --- a/trace_decoder/src/processed_block_trace.rs +++ b/trace_decoder/src/processed_block_trace.rs @@ -4,6 +4,7 @@ use std::iter::once; use ethereum_types::{Address, H256, U256}; use evm_arithmetization::generation::mpt::{AccountRlp, LegacyReceiptRlp}; +use evm_arithmetization::JumpDestTableWitness; use mpt_trie::nibbles::Nibbles; use crate::hash; @@ -188,6 +189,7 @@ impl TxnInfo { txn_bytes, receipt_node_bytes, gas_used: self.meta.gas_used, + jumpdest_table: self.meta.jumpdest_table, }; ProcessedTxnInfo { @@ -241,4 +243,5 @@ pub(crate) struct TxnMetaState { pub(crate) txn_bytes: Option>, pub(crate) receipt_node_bytes: Vec, pub(crate) gas_used: u64, + pub(crate) jumpdest_table: JumpDestTableWitness, } diff --git a/zero_bin/common/Cargo.toml b/zero_bin/common/Cargo.toml index 9ee2ac5fe..f1c37de53 100644 --- a/zero_bin/common/Cargo.toml +++ b/zero_bin/common/Cargo.toml @@ -22,3 +22,4 @@ futures = { workspace = true } tokio = { workspace = true } alloy = { workspace = true } async-stream = { workspace = true } +pretty_assertions = "1.4.0" diff --git a/zero_bin/rpc/Cargo.toml b/zero_bin/rpc/Cargo.toml index 6724c16fe..f4c408485 100644 --- a/zero_bin/rpc/Cargo.toml +++ b/zero_bin/rpc/Cargo.toml @@ -13,9 +13,11 @@ __compat_primitive_types = { workspace = true } alloy.workspace = true anyhow = { workspace = true } clap = { workspace = true } +env_logger = { workspace = true } evm_arithmetization = { workspace = true } futures = { workspace = true } hex = { workspace = true } +log = { workspace = true } lru = { workspace = true } mpt_trie = { workspace = true } serde = { workspace = true } diff --git a/zero_bin/rpc/src/native/mod.rs b/zero_bin/rpc/src/native/mod.rs index 892a799d6..7d57b398a 100644 --- a/zero_bin/rpc/src/native/mod.rs +++ b/zero_bin/rpc/src/native/mod.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use alloy::{ primitives::B256, providers::Provider, @@ -15,8 +13,6 @@ use crate::provider::CachedProvider; mod state; mod txn; -type CodeDb = HashMap<__compat_primitive_types::H256, Vec>; - /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( provider: &CachedProvider, diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 61bcf5f4a..ab9113eeb 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -1,31 +1,40 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + sync::OnceLock, +}; use __compat_primitive_types::{H256, U256}; use alloy::{ - primitives::{keccak256, Address, B256}, + primitives::{keccak256, Address, B256, U160}, providers::{ ext::DebugApi as _, network::{eip2718::Encodable2718, Ethereum, Network}, Provider, }, rpc::types::{ - eth::Transaction, - eth::{AccessList, Block}, + eth::{AccessList, Block, Transaction}, trace::geth::{ - AccountState, DiffMode, GethDebugBuiltInTracerType, GethTrace, PreStateConfig, + AccountState, DefaultFrame, DiffMode, GethDebugBuiltInTracerType, GethDebugTracerType, + GethDebugTracingOptions, GethDefaultTracingOptions, GethTrace, PreStateConfig, PreStateFrame, PreStateMode, }, - trace::geth::{GethDebugTracerType, GethDebugTracingOptions}, }, transports::Transport, }; use anyhow::Context as _; +use evm_arithmetization::{CodeDb, JumpDestTableWitness}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use super::CodeDb; use crate::Compat; +/// Provides a way to check in constant time if an address points to a precompile. +fn precompiles() -> &'static HashSet
{ + static PRECOMPILES: OnceLock> = OnceLock::new(); + PRECOMPILES + .get_or_init(|| HashSet::
::from_iter((0..9).map(|x| Address::from(U160::from(x))))) +} + /// Processes the transactions in the given block and updates the code db. pub(super) async fn process_transactions( block: &Block, @@ -63,17 +72,12 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace) = fetch_tx_data(provider, &tx.hash).await?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + fetch_tx_data(provider, &tx.hash).await?; + let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let tx_meta = TxnMeta { - byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), - new_txn_trie_node_byte: vec![], - new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), - gas_used: tx_receipt.gas_used as u64, - }; - let (code_db, tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), @@ -82,6 +86,21 @@ where _ => unreachable!(), }; + let jumpdest_table: JumpDestTableWitness = + if let GethTrace::Default(structlog_frame) = structlog_trace { + generate_jumpdest_table(tx, &structlog_frame, &tx_traces).await? + } else { + unreachable!() + }; + + let tx_meta = TxnMeta { + byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), + new_txn_trie_node_byte: vec![], + new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), + gas_used: tx_receipt.gas_used as u64, + jumpdest_table, + }; + Ok(( code_db, TxnInfo { @@ -98,7 +117,15 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, -) -> anyhow::Result<(::ReceiptResponse, GethTrace, GethTrace), anyhow::Error> +) -> anyhow::Result< + ( + ::ReceiptResponse, + GethTrace, + GethTrace, + GethTrace, + ), + anyhow::Error, +> where ProviderT: Provider, TransportT: Transport + Clone, @@ -106,14 +133,21 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); + let structlog_trace_fut = + provider.debug_trace_transaction(*tx_hash, structlog_tracing_options()); - let (tx_receipt, pre_trace, diff_trace) = - futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( + tx_receipt_fut, + pre_trace_fut, + diff_trace_fut, + structlog_trace_fut + )?; Ok(( tx_receipt.context("Transaction receipt not found.")?, pre_trace, diff_trace, + structlog_trace, )) } @@ -163,6 +197,7 @@ async fn process_tx_traces( let balance = post_state.and_then(|x| x.balance.map(Compat::compat)); let (storage_read, storage_written) = process_storage( + // Q: doesn't this remove the address from warm addresses? access_list.remove(&address).unwrap_or_default(), read_state, post_state, @@ -183,7 +218,6 @@ async fn process_tx_traces( traces.insert(address, result); } - Ok((code_db, traces)) } @@ -270,7 +304,6 @@ async fn process_code( (_, Some(read_code)) => { let code_hash = keccak256(read_code).compat(); code_db.insert(code_hash, read_code.to_vec()); - Some(ContractCodeUsage::Read(code_hash)) } _ => None, @@ -340,3 +373,115 @@ fn prestate_tracing_options(diff_mode: bool) -> GethDebugTracingOptions { ..GethDebugTracingOptions::default() } } + +/// Tracing options for the debug_traceTransaction call used for filling +/// JumpDest tables. +fn structlog_tracing_options() -> GethDebugTracingOptions { + GethDebugTracingOptions { + config: GethDefaultTracingOptions { + disable_stack: Some(false), + disable_memory: Some(true), + disable_storage: Some(true), + ..GethDefaultTracingOptions::default() + }, + tracer: None, + ..GethDebugTracingOptions::default() + } +} + +async fn generate_jumpdest_table( + tx: &Transaction, + structlog_trace: &DefaultFrame, + tx_traces: &HashMap, +) -> anyhow::Result { + let mut jumpdest_table = JumpDestTableWitness::default(); + + if structlog_trace.struct_logs.len() == 0 { + return Ok(jumpdest_table); + }; + + let callee_addr_to_code_hash: HashMap = tx_traces + .iter() + .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) + .filter(|(_callee_addr, code_usage)| code_usage.is_some()) + .map(|(callee_addr, code_usage)| { + (*callee_addr, code_usage.as_ref().unwrap().get_code_hash()) + }) + .collect(); + + let to_address: Address = tx + .to + .expect(&format!("No `to`-address for tx: {}.", tx.hash)); + + // Guard against transactions to a non-contract address. + if !callee_addr_to_code_hash.contains_key(&to_address) { + return Ok(jumpdest_table); + } + let entrypoint_code_hash: H256 = callee_addr_to_code_hash[&to_address]; + + // The next available context. Starts at 1. Never decrements. + let mut next_ctx_available = 1; + // Immediately use context 1; + let mut call_stack = vec![(entrypoint_code_hash, next_ctx_available)]; + next_ctx_available += 1; + + for entry in structlog_trace.struct_logs.iter() { + debug_assert!(entry.depth as usize <= next_ctx_available); + log::debug!("{}", entry.op.as_str()); + match entry.op.as_str() { + "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + let callee_address = { + // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 + let callee_raw = *entry + .stack + .as_ref() + .expect("No stack found in structLog.") + .iter() + .rev() + .skip(1) + .next() + .expect("Stack must contain at least two values for a CALL instruction."); + let lower_bytes = U160::from(callee_raw); + let callee_address = Address::from(lower_bytes); + callee_address + }; + + if precompiles().contains(&callee_address) { + log::debug!("PRECOMPILE at address {} called.", &callee_address); + } else { + if callee_addr_to_code_hash.contains_key(&callee_address) { + let code_hash = callee_addr_to_code_hash[&callee_address]; + call_stack.push((code_hash, next_ctx_available)); + } else { + log::debug!( + "Callee address {} has no associated `code_hash`. Please verify that this is not an EOA.", + &callee_address + ); + } + } + next_ctx_available += 1; + } + "JUMPDEST" => { + let (code_hash, ctx) = call_stack + .last() + .expect("Call stack was empty when a JUMPDEST was encountered."); + jumpdest_table + .0 + .entry(*code_hash) + .or_default() + .0 + .entry(*ctx) + .or_default() + .insert(entry.pc as usize); + } + "EXTCODECOPY" | "EXTCODESIZE" => { + next_ctx_available += 1; + } + "RETURN" => { + call_stack.pop().expect("Call stack was empty at POP."); + } + _ => (), + } + } + Ok(jumpdest_table) +} diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index 76af3b0dc..1a91436bc 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -33,7 +33,7 @@ export RUST_BACKTRACE=full export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 @@ -89,6 +89,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then + echo $RUSTFLAGS cargo run --release --features test_only --bin leader -- --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof"