Benchmarks for VM and Native

Makefile

@@ -0,0 +1,4 @@
+.PHONY: bench
+bench:
+	cargo b --release --bin snforge
+	./scripts/benchmarks.sh $(SKIP)

crates/forge-runner/src/forge_config.rs

@@ -27,6 +27,7 @@ pub struct TestRunnerConfig {
     pub environment_variables: HashMap<String, String>,
     pub tracked_resource: ForgeTrackedResource,
     pub experimental_oracles: bool,
+    pub use_native: bool,
 }
 
 #[derive(Debug, PartialEq)]

crates/forge-runner/src/running.rs

@@ -81,18 +81,21 @@ pub fn run_test(
             return TestCaseSummary::Interrupted {};
         }
 
-        let run_result = run_native_test_case(
-            &case,
-            &_aot_executor,
-            &RuntimeConfig::from(&forge_config.test_runner_config),
-            None,
-        );
-        // let run_result = run_test_case(
-        //     &case,
-        //     &_casm_program,
-        //     &RuntimeConfig::from(&forge_config.test_runner_config),
-        //     None,
-        // );
+        let run_result = if forge_config.test_runner_config.use_native {
+            run_native_test_case(
+                &case,
+                &_aot_executor,
+                &RuntimeConfig::from(&forge_config.test_runner_config),
+                None,
+            )
+        } else {
+            run_test_case(
+                &case,
+                &_casm_program,
+                &RuntimeConfig::from(&forge_config.test_runner_config),
+                None,
+            )
+        };
 
         if send.is_closed() {
             return TestCaseSummary::Interrupted {};

crates/forge/src/combine_configs.rs

@@ -29,6 +29,7 @@ pub fn combine_configs(
     additional_args: &[OsString],
     trace_args: TraceArgs,
     experimental_oracles: bool,
+    use_native: bool,
 ) -> ForgeConfig {
     let execution_data_to_save = ExecutionDataToSave::from_flags(
         save_trace_data || forge_config_from_scarb.save_trace_data,
@@ -53,6 +54,7 @@ pub fn combine_configs(
             tracked_resource,
             environment_variables: env::vars().collect(),
             experimental_oracles,
+            use_native,
         }),
         output_config: Arc::new(OutputConfig {
             trace_args,
@@ -84,6 +86,7 @@ mod tests {
             &[],
             TraceArgs::default(),
             false,
+            false,
         );
         let config2 = combine_configs(
             false,
@@ -101,6 +104,7 @@ mod tests {
             &[],
             TraceArgs::default(),
             false,
+            false,
         );
 
         assert_ne!(config.test_runner_config.fuzzer_seed, 0);
@@ -129,6 +133,7 @@ mod tests {
             &[],
             TraceArgs::default(),
             false,
+            false,
         );
         assert_eq!(
             config,
@@ -144,6 +149,7 @@ mod tests {
                     contracts_data: ContractsData::default(),
                     environment_variables: config.test_runner_config.environment_variables.clone(),
                     experimental_oracles: false,
+                    use_native: false
                 }),
                 output_config: Arc::new(OutputConfig {
                     detailed_resources: false,
@@ -185,6 +191,7 @@ mod tests {
             &[],
             TraceArgs::default(),
             false,
+            false,
         );
         assert_eq!(
             config,
@@ -200,6 +207,7 @@ mod tests {
                     contracts_data: ContractsData::default(),
                     environment_variables: config.test_runner_config.environment_variables.clone(),
                     experimental_oracles: false,
+                    use_native: false
                 }),
                 output_config: Arc::new(OutputConfig {
                     detailed_resources: true,
@@ -245,6 +253,7 @@ mod tests {
             &[],
             TraceArgs::default(),
             false,
+            false,
         );
 
         assert_eq!(
@@ -261,6 +270,7 @@ mod tests {
                     contracts_data: ContractsData::default(),
                     environment_variables: config.test_runner_config.environment_variables.clone(),
                     experimental_oracles: false,
+                    use_native: false
                 }),
                 output_config: Arc::new(OutputConfig {
                     detailed_resources: true,

crates/forge/src/lib.rs

@@ -220,6 +220,9 @@ pub struct TestArgs {
     #[arg(long, default_value_t = false, env = "SNFORGE_EXPERIMENTAL_ORACLES")]
     experimental_oracles: bool,
 
+    #[arg(long, default_value_t = false)]
+    use_native: bool,
+
     #[command(flatten)]
     scarb_args: ScarbArgs,
 }
@@ -274,7 +277,6 @@ pub enum ExitStatus {
 
 pub fn main_execution(ui: Arc<UI>) -> Result<ExitStatus> {
     let cli = Cli::parse();
-
     match cli.subcommand {
         ForgeSubcommand::Init { name } => {
             init::init(name.as_str(), &ui)?;

crates/forge/src/run_tests/package.rs

@@ -108,6 +108,7 @@ impl RunForPackageArgs {
             &args.additional_args,
             args.trace_args.clone(),
             args.experimental_oracles,
+            args.use_native,
         ));
 
         let test_filter = TestsFilter::from_flags(

crates/forge/test_utils/src/running_tests.rs

@@ -79,6 +79,7 @@ pub fn run_test_case(
                     tracked_resource,
                     environment_variables: test.env().clone(),
                     experimental_oracles: false,
+                    use_native: false,
                 }),
                 output_config: Arc::new(OutputConfig {
                     detailed_resources: false,

crates/forge/tests/integration/setup_fork.rs

@@ -161,6 +161,7 @@ fn fork_aliased_decorator() {
                         tracked_resource: ForgeTrackedResource::CairoSteps,
                         environment_variables: test.env().clone(),
                         experimental_oracles: false,
+                        use_native: false,
                     }),
                     output_config: Arc::new(OutputConfig {
                         detailed_resources: false,
@@ -254,6 +255,7 @@ fn fork_aliased_decorator_overrding() {
                         tracked_resource: ForgeTrackedResource::CairoSteps,
                         environment_variables: test.env().clone(),
                         experimental_oracles: false,
+                        use_native: false,
                     }),
                     output_config: Arc::new(OutputConfig {
                         detailed_resources: false,

scarb_programs/cairo_program/Scarb.toml

@@ -0,0 +1,48 @@
+[package]
+name = "cairo_program"
+version = "0.1.0"
+edition = "2024_07"
+
+# See more keys and their definitions at https://docs.swmansion.com/scarb/docs/reference/manifest.html
+
+[dependencies]
+
+[dev-dependencies]
+snforge_std_deprecated = "0.48.0"
+assert_macros = "2.11.2"
+
+[scripts]
+test = "snforge test"
+
+[tool.scarb]
+allow-prebuilt-plugins = ["snforge_std_deprecated"]
+
+# Visit https://foundry-rs.github.io/starknet-foundry/appendix/scarb-toml.html for more information
+
+# [tool.snforge]                                             # Define `snforge` tool section
+# exit_first = true                                          # Stop tests execution immediately upon the first failure
+# fuzzer_runs = 1234                                         # Number of runs of the random fuzzer
+# fuzzer_seed = 1111                                         # Seed for the random fuzzer
+
+# [[tool.snforge.fork]]                                      # Used for fork testing
+# name = "SOME_NAME"                                         # Fork name
+# url = "http://your.rpc.url"                                # Url of the RPC provider
+# block_id.tag = "latest"                                    # Block to fork from (block tag)
+
+# [[tool.snforge.fork]]
+# name = "SOME_SECOND_NAME"
+# url = "http://your.second.rpc.url"                         
+# block_id.number = "123"                                    # Block to fork from (block number)
+
+# [[tool.snforge.fork]]
+# name = "SOME_THIRD_NAME"
+# url = "http://your.third.rpc.url"
+# block_id.hash = "0x123"                                    # Block to fork from (block hash)
+
+# [profile.dev.cairo]                                        # Configure Cairo compiler
+# unstable-add-statements-code-locations-debug-info = true   # Should be used if you want to use coverage
+# unstable-add-statements-functions-debug-info = true        # Should be used if you want to use coverage/profiler
+# inlining-strategy = "avoid"                                # Should be used if you want to use coverage
+
+# [features]                                                 # Used for conditional compilation
+# enable_for_tests = []                                      # Feature name and list of other features that should be enabled with it

scarb_programs/cairo_program/snfoundry.toml

@@ -0,0 +1,11 @@
+# Visit https://foundry-rs.github.io/starknet-foundry/appendix/snfoundry-toml.html
+# and https://foundry-rs.github.io/starknet-foundry/projects/configuration.html for more information
+
+# [sncast.default]                                         # Define a profile name
+# url = "https://starknet-sepolia.public.blastapi.io/rpc/v0_8" # Url of the RPC provider
+# accounts-file = "../account-file"                        # Path to the file with the account data
+# account = "mainuser"                                     # Account from `accounts_file` or default account file that will be used for the transactions
+# keystore = "~/keystore"                                  # Path to the keystore file
+# wait-params = { timeout = 300, retry-interval = 10 }     # Wait for submitted transaction parameters
+# block-explorer = "StarkScan"                             # Block explorer service used to display links to transaction details
+# show-explorer-links = true                               # Print links pointing to pages with transaction details in the chosen block explorer

scarb_programs/cairo_program/src/lib.cairo

@@ -0,0 +1,123 @@
+use core::dict::Felt252Dict;
+
+fn main() -> felt252 {
+    factorial(1, 1)
+}
+
+fn factorial(value: felt252, n: felt252) -> felt252 {
+    if (n == 1) {
+        value
+    } else {
+        factorial(value * n, n - 1)
+    }
+}
+
+fn init_dict(length: u64) -> Felt252Dict<felt252> {
+    let mut balances: Felt252Dict<felt252> = Default::default();
+
+    for i in 0..length {
+        let x: felt252 = i.into();
+        balances.insert(x, x);
+    };
+
+    return balances;
+}
+
+fn fib(a: felt252, b: felt252, n: felt252) -> felt252 {
+    match n {
+        0 => a,
+        _ => fib(b, a + b, n - 1),
+    }
+}
+
+fn search(array: @Array<felt252>, number: felt252) -> u32 {
+    let mut index = 0;
+
+    while index < array.len() {
+        if *array[index] == number {
+            break;
+        }
+
+        index += 1;
+    };
+
+    return index;
+}
+
+fn init_array(length: u32) -> Array<felt252> {
+    let mut array = ArrayTrait::new();
+    for i in 0..length {
+        array.append(i.into());
+    };
+
+    return array;
+}
+
+// Logistic map implementation.
+//
+// The value would normally be a number between zero and one, however since floats are not
+// available, zero corresponds to a felt zero and one to PRIME - 1.
+fn iterate_map(r: felt252, x: felt252) -> felt252 {
+    r * x * -x
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{factorial, init_dict, fib, init_array, search, iterate_map};
+
+    #[test]
+    fn test_factorial() {
+        assert_eq!(factorial(1, 2), 2, "invalid result");
+        assert_eq!(factorial(1, 3), 6, "invalid result");
+        assert_eq!(factorial(1, 4), 24, "invalid result");
+        assert_eq!(factorial(1, 10), 3628800, "invalid result");
+        assert_eq!(factorial(1, 2000000), 0x4d6e41de886ac83938da3456ccf1481182687989ead34d9d35236f0864575a0, "invalid result");
+    }
+
+    #[test]
+    fn test_dict() {
+        let mut dict = init_dict(1000001);
+        let last = dict.get(1000000);
+        assert_eq!(last, 1000000, "invalid result");
+    }
+
+    #[test]
+    fn test_fibonacci() {
+        assert_eq!(fib(0, 1, 10), 55, "invalid result");
+        assert_eq!(fib(0, 1, 2000000), 0x79495858064f7881b9eff3a923642b2990b5a4342da5470eb2251df58d9acfb, "invalid result");
+    }
+
+    #[test]
+    fn test_linear_search() {
+        let array = init_array(4001);
+
+        let index = search(@array, 1000);
+        assert_eq!(index, 1000, "invalid result");
+        let index = search(@array, 2000);
+        assert_eq!(index, 2000, "invalid result");
+        let index = search(@array, 3000);
+        assert_eq!(index, 3000, "invalid result");
+        let index = search(@array, 4000);
+        assert_eq!(index, 4000, "invalid result");
+    }
+
+    #[test]
+    fn test_map_iteration() {
+        // Initial value.
+        let mut x = 1234567890123456789012345678901234567890;
+
+        // Iterate the map.
+        let mut i = 15000;
+        let result = loop {
+            x = iterate_map(4, x);
+
+            if i == 0 {
+                break x;
+            }
+
+            i = i - 1;
+        };
+
+        assert_eq!(result, 0x12d35a3ae9fe7c56f194b12b34d567a844432acd2b7da993a158c15447a424d, "invalid result");
+    }
+}

scripts/benchmarks.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+PROGRAMS_DIR="scarb_programs/"
+SKIP=$1
+
+run_bench() {
+    native_command='../../target/release/snforge test --use-native'
+    vm_command='../../target/release/snforge test --max-n-steps 100000000'
+
+    # If an argument was passed, it means we should use it as the string for the skip argument in snforge
+    if [ -n "$SKIP" ]; then
+      native_command+=" --skip $SKIP"
+      vm_command+=" --skip $SKIP"
+    fi
+
+    hyperfine \
+        --warmup 5 \
+        "$native_command" \
+        "$vm_command"
+}
+
+for dir in "$PROGRAMS_DIR"/*/; do
+  (
+    cd "$dir"
+    run_bench
+  )
+done
+