diff --git a/bins/revme/src/cmd/semantics.rs b/bins/revme/src/cmd/semantics.rs index 463132b0..6aa26290 100644 --- a/bins/revme/src/cmd/semantics.rs +++ b/bins/revme/src/cmd/semantics.rs @@ -9,12 +9,14 @@ use log::{error, info, LevelFilter}; use rayon::prelude::*; use state::AccountInfo; use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; use clap::{ArgAction, Parser}; mod errors; pub use errors::Errors; +use errors::SkipReason; mod semantic_tests; use semantic_tests::SemanticTests; mod compiler_evm_versions; @@ -28,6 +30,41 @@ use utils::find_test_files; use crate::cmd::semantics::test_cases::TestCase; +/// Thread-safe counters for tracking skipped test files by reason. +struct SkipCounts { + counts: [AtomicUsize; 7], +} + +impl SkipCounts { + fn new() -> Self { + Self { + counts: std::array::from_fn(|_| AtomicUsize::new(0)), + } + } + + fn increment(&self, reason: SkipReason) { + self.counts[reason.index()].fetch_add(1, Ordering::Relaxed); + } + + fn print_summary(&self) { + let entries: Vec<(usize, SkipReason)> = SkipReason::ALL + .iter() + .map(|&r| (self.counts[r.index()].load(Ordering::Relaxed), r)) + .filter(|(count, _)| *count > 0) + .collect(); + + let total: usize = entries.iter().map(|(c, _)| *c).sum(); + if total == 0 { + return; + } + + println!("Skipped {} test file(s):", total); + for (count, reason) in &entries { + println!(" {:>4} - {}", count, reason); + } + } +} + /// EVM runner command that allows running Solidity semantic tests. /// If a path is provided, it will process that file or recursively process all `.sol` files in that directory. /// If no path is provided, it defaults to the Solidity semantic tests directory. @@ -74,18 +111,19 @@ impl Cmd { let start_time = Instant::now(); let test_files = self.find_test_files()?; let n_files = test_files.len(); + let skip_counts = SkipCounts::new(); let failures = match self.single_thread { true => { info!("Running in single-threaded mode"); - self.run_single_threaded(test_files) + self.run_single_threaded(test_files, &skip_counts) } false => { info!("Running in multi-threaded mode"); test_files .par_iter() .filter_map(|test_file| { - match self.process_test_file(test_file.clone()) { + match self.process_test_file(test_file.clone(), &skip_counts) { Err(file_failures) => Some((test_file.clone(), file_failures)), Ok(_) => None, // No failures for this file } @@ -96,6 +134,9 @@ impl Cmd { let duration = start_time.elapsed(); info!("Execution time: {:?}", duration); + + skip_counts.print_summary(); + if failures.len() == 0 { println!("All tests passed across {} files ✅", n_files); return Ok(()); @@ -149,10 +190,11 @@ impl Cmd { fn run_single_threaded( &self, test_files: Vec, + skip_counts: &SkipCounts, ) -> Vec<(PathBuf, Vec<(Errors, Option)>)> { let mut failures = vec![]; for test_file in test_files { - if let Err(file_failures) = self.process_test_file(test_file.clone()) { + if let Err(file_failures) = self.process_test_file(test_file.clone(), skip_counts) { failures.push((test_file, file_failures)); if !self.keep_going { return failures; @@ -209,7 +251,11 @@ impl Cmd { } } - fn process_test_file(&self, test_file: PathBuf) -> Result<(), Vec<(Errors, Option)>> { + fn process_test_file( + &self, + test_file: PathBuf, + skip_counts: &SkipCounts, + ) -> Result<(), Vec<(Errors, Option)>> { info!("test_file: {:?}", test_file); let test_file_path = match test_file.to_str() { Some(p) => p, @@ -259,7 +305,8 @@ impl Cmd { } failures } - Err(Errors::UnhandledTestFormat) => { + Err(Errors::Skipped(reason)) => { + skip_counts.increment(reason); return Ok(()); } Err(e) => { diff --git a/bins/revme/src/cmd/semantics/errors.rs b/bins/revme/src/cmd/semantics/errors.rs index 5e42528f..8dc9dc53 100644 --- a/bins/revme/src/cmd/semantics/errors.rs +++ b/bins/revme/src/cmd/semantics/errors.rs @@ -1,8 +1,71 @@ +use std::fmt; use std::io::Error as IoError; use primitives::Bytes; use crate::cmd::semantics::test_cases::TestCase; + +/// Reason a test file was skipped during processing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + /// Test uses multi-source layout (`==== Source:`) + MultiSource, + /// Test requires `allowNonExistingFunctions: true` + NonExistingFunctions, + /// Test requires `revertStrings: debug` + DebugRevertStrings, + /// Test requires via-IR but `--unsafe-via-ir` was not passed + ViaIrUnsafeRequired, + /// Test requires via-IR but `--skip-via-ir` was passed + ViaIrSkipped, + /// Test opts out of via-IR (`compileViaYul: false`) but `--via-ir` is forced + ViaIrOptOut, + /// Test requires EOF but `--eof` was not passed + EofNotEnabled, +} + +impl fmt::Display for SkipReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MultiSource => write!(f, "unsupported: multi-source files (==== Source:)"), + Self::NonExistingFunctions => write!(f, "unsupported: allowNonExistingFunctions"), + Self::DebugRevertStrings => write!(f, "unsupported: revertStrings: debug"), + Self::ViaIrUnsafeRequired => { + write!(f, "via-IR required (use --unsafe-via-ir to enable)") + } + Self::ViaIrSkipped => write!(f, "via-IR required (excluded by --skip-via-ir)"), + Self::ViaIrOptOut => write!(f, "via-IR opt-out (excluded by --via-ir)"), + Self::EofNotEnabled => write!(f, "EOF required (use --eof to enable)"), + } + } +} + +impl SkipReason { + /// All variants, used for iterating when printing summaries. + pub const ALL: [SkipReason; 7] = [ + Self::MultiSource, + Self::NonExistingFunctions, + Self::DebugRevertStrings, + Self::ViaIrUnsafeRequired, + Self::ViaIrSkipped, + Self::ViaIrOptOut, + Self::EofNotEnabled, + ]; + + /// Stable index for use with `SkipCounts`. + pub fn index(self) -> usize { + match self { + Self::MultiSource => 0, + Self::NonExistingFunctions => 1, + Self::DebugRevertStrings => 2, + Self::ViaIrUnsafeRequired => 3, + Self::ViaIrSkipped => 4, + Self::ViaIrOptOut => 5, + Self::EofNotEnabled => 6, + } + } +} + #[derive(Debug, thiserror::Error)] pub enum Errors { #[error("The specified path does not exist")] @@ -23,8 +86,8 @@ pub enum Errors { Io(#[from] IoError), #[error("Invalid Test Format")] InvalidTestFormat, - #[error("Unhandled Test Format: === Source:")] - UnhandledTestFormat, + #[error("Skipped: {0}")] + Skipped(SkipReason), #[error("Invalid function signature")] InvalidFunctionSignature, #[error("Invalid Test Output")] diff --git a/bins/revme/src/cmd/semantics/semantic_tests.rs b/bins/revme/src/cmd/semantics/semantic_tests.rs index af67bc0a..4dd4ba8e 100644 --- a/bins/revme/src/cmd/semantics/semantic_tests.rs +++ b/bins/revme/src/cmd/semantics/semantic_tests.rs @@ -7,18 +7,13 @@ use revm::primitives::{hex, Address, Bytes}; use super::{ compiler_evm_versions::EVMVersion, + errors::SkipReason, solc_config::SolcArgs, test_cases::TestCase, utils::{extract_compile_via_yul, extract_functions_from_source, needs_eof}, Errors, }; -const SKIP_KEYWORD: [&str; 3] = [ - "==== Source:", - "allowNonExistingFunctions: true", - "revertStrings: debug", -]; - #[derive(Debug, Clone)] pub struct ContractInfo { pub contract_name: String, @@ -121,23 +116,43 @@ impl SemanticTests { return Err(Errors::InvalidTestFormat); } - // Early exit if the content contains `==== Source:` We do not handle this yet nor - // nonExistingFunctions nor Libraries that generate some slightly different Bytecode with - // the unhandled "_" - if SKIP_KEYWORD - .iter() - .any(|&keyword| content.contains(keyword)) - { - return Err(Errors::UnhandledTestFormat); + // Skip unsupported test formats individually so we can track counts per reason. + if content.contains("==== Source:") { + return Err(Errors::Skipped(SkipReason::MultiSource)); + } + if content.contains("allowNonExistingFunctions: true") { + return Err(Errors::Skipped(SkipReason::NonExistingFunctions)); + } + if content.contains("revertStrings: debug") { + return Err(Errors::Skipped(SkipReason::DebugRevertStrings)); } let expectations = parts[1].to_string(); let evm_version = EVMVersion::extract(&content); - let via_ir = extract_compile_via_yul(&content); + let compile_via_yul = extract_compile_via_yul(&content); + // If --skip-via-ir is set, skip tests that require via-IR. + if compile_via_yul == Some(true) && solc_args.skip_via_ir { + return Err(Errors::Skipped(SkipReason::ViaIrSkipped)); + } + // Auto-skip: tests needing via-IR require --unsafe-via-ir on the runner, + // because the compiler will reject --via-ir without --unsafe-via-ir. + if compile_via_yul == Some(true) && !solc_args.unsafe_via_ir { + return Err(Errors::Skipped(SkipReason::ViaIrUnsafeRequired)); + } + // If the test explicitly opts out of via-IR and we're forcing --via-ir, skip it. + if compile_via_yul == Some(false) && solc_args.via_ir { + return Err(Errors::Skipped(SkipReason::ViaIrOptOut)); + } + let via_ir = compile_via_yul.unwrap_or(false) || solc_args.via_ir; let eof_mode = needs_eof(&content); - if eof_mode & skip_eof { - return Err(Errors::UnhandledTestFormat); + // EOF implicitly needs --via-ir, so also needs --unsafe-via-ir. + if eof_mode && (skip_eof || !solc_args.unsafe_via_ir) { + return Err(Errors::Skipped(if skip_eof { + SkipReason::EofNotEnabled + } else { + SkipReason::ViaIrUnsafeRequired + })); } let mut contract_infos = Self::get_contract_infos( @@ -175,9 +190,14 @@ impl SemanticTests { solc.arg("--evm-version").arg(v.to_string()); } - // via‑IR is required for EOF; keep explicit flag for legacy tests + // via-IR is required for EOF; keep explicit flag for legacy tests. + // The seismic compiler requires --unsafe-via-ir whenever --via-ir is used. if via_ir || eof_mode { + if !solc_args.unsafe_via_ir { + return Err(Errors::Skipped(SkipReason::ViaIrUnsafeRequired)); + } solc.arg("--via-ir"); + solc.arg("--unsafe-via-ir"); } if eof_mode { diff --git a/bins/revme/src/cmd/semantics/solc_config.rs b/bins/revme/src/cmd/semantics/solc_config.rs index a27651cd..963902c2 100644 --- a/bins/revme/src/cmd/semantics/solc_config.rs +++ b/bins/revme/src/cmd/semantics/solc_config.rs @@ -10,4 +10,51 @@ pub struct SolcArgs { /// Set the runs parameter for optimizer (requires --optimize) #[clap(long)] pub optimizer_runs: Option, + + /// Compile via Yul intermediate representation (--via-ir). + /// + /// Forces all tests through solc's via-IR pipeline, UNLESS a test + /// explicitly opts out with `// compileViaYul: false` — in which + /// case the test is skipped entirely. + /// + /// Requires --unsafe-via-ir (the seismic solc compiler requires + /// --unsafe-via-ir whenever --via-ir is used). + /// + /// Mutually exclusive with --skip-via-ir. + /// + /// compileViaYul | --via-ir | --skip-via-ir | --unsafe-via-ir | result + /// ───────────────+──────────+───────────────+─────────────────+────────────────────────────── + /// true | off | off | off | SKIP (needs --unsafe-via-ir) + /// true | off | off | on | compile with --via-ir --unsafe-via-ir + /// true | on* | — | on | compile with --via-ir --unsafe-via-ir + /// true | — | on | — | SKIP + /// false | off | any | any | compile without --via-ir + /// false | on* | — | on | SKIP + /// (not set) | off | any | any | compile without --via-ir + /// (not set) | on* | — | on | compile with --via-ir --unsafe-via-ir + /// + /// *--via-ir requires --unsafe-via-ir (clap validation). + /// + /// Commonly combined with --optimize. Without --optimize, via-IR + /// can produce inefficient bytecode and "stack too deep" errors. + /// Example: --via-ir --unsafe-via-ir --optimize --optimizer-runs 200 + #[clap(long, conflicts_with = "skip_via_ir", requires = "unsafe_via_ir")] + pub via_ir: bool, + + /// Skip tests that require via-IR compilation (compileViaYul: true). + /// + /// Use this when the solc binary does not support --via-ir. + /// Mutually exclusive with --via-ir and --unsafe-via-ir. + #[clap(long, conflicts_with = "via_ir")] + pub skip_via_ir: bool, + + /// Allow via-IR compilation by also passing --unsafe-via-ir to the compiler. + /// + /// The seismic solc compiler requires this flag whenever --via-ir is used. + /// Without this flag, tests that require via-IR (compileViaYul: true) are + /// automatically skipped. + /// + /// Mutually exclusive with --skip-via-ir. + #[clap(long, conflicts_with = "skip_via_ir")] + pub unsafe_via_ir: bool, } diff --git a/bins/revme/src/cmd/semantics/utils.rs b/bins/revme/src/cmd/semantics/utils.rs index d16dd4a0..097558e0 100644 --- a/bins/revme/src/cmd/semantics/utils.rs +++ b/bins/revme/src/cmd/semantics/utils.rs @@ -93,18 +93,18 @@ pub(crate) fn find_test_files(dir: &Path) -> Result, Errors> { Ok(test_files) } -pub(crate) fn extract_compile_via_yul(content: &str) -> bool { +pub(crate) fn extract_compile_via_yul(content: &str) -> Option { let parts: Vec<&str> = content.split("// ====").collect(); if parts.len() < 2 { - return false; + return None; } for line in parts[1].lines() { if let Some(flag_part) = line.trim().strip_prefix("// compileViaYul:") { - return flag_part.trim() == "true"; + return Some(flag_part.trim() == "true"); } } - false + None } pub(crate) fn needs_eof(content: &str) -> bool {