split rust code into crates

so that we get more parallelism out of cargo

Author: folkertdev

crates/intrinsic-test/src/arm/mod.rs

@@ -13,7 +13,9 @@ use crate::common::SupportedArchitectureTest;
 use crate::common::cli::ProcessedCli;
 use crate::common::compare::compare_outputs;
 use crate::common::gen_c::{write_main_cpp, write_mod_cpp};
-use crate::common::gen_rust::{compile_rust_programs, write_cargo_toml, write_main_rs};
+use crate::common::gen_rust::{
+    compile_rust_programs, write_bin_cargo_toml, write_lib_cargo_toml, write_lib_rs, write_main_rs,
+};
 use crate::common::intrinsic::Intrinsic;
 use crate::common::intrinsic_helpers::TypeKind;
 use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, build_notices};
@@ -125,19 +127,17 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
             "aarch64"
         };
 
-        let available_parallelism = std::thread::available_parallelism().unwrap().get();
-        let chunk_size = self.intrinsics.len().div_ceil(available_parallelism);
+        let (chunk_size, chunk_count) = chunk_info(self.intrinsics.len());
 
         let mut cargo = File::create("rust_programs/Cargo.toml").unwrap();
-        write_cargo_toml(&mut cargo, &[]).unwrap();
+        write_bin_cargo_toml(&mut cargo, chunk_count).unwrap();
 
         let mut main_rs = File::create("rust_programs/src/main.rs").unwrap();
         write_main_rs(
             &mut main_rs,
-            available_parallelism,
-            architecture,
+            chunk_count,
             AARCH_CONFIGURATIONS,
-            F16_FORMATTING_DEF,
+            "",
             self.intrinsics.iter().map(|i| i.name.as_str()),
         )
         .unwrap();
@@ -151,20 +151,21 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
             .par_chunks(chunk_size)
             .enumerate()
             .map(|(i, chunk)| {
-                use std::io::Write;
+                std::fs::create_dir_all(format!("rust_programs/mod_{i}/src"))?;
 
-                let rust_filename = format!("rust_programs/src/mod_{i}.rs");
+                let rust_filename = format!("rust_programs/mod_{i}/src/lib.rs");
                 trace!("generating `{rust_filename}`");
-                let mut file = File::create(rust_filename).unwrap();
+                let mut file = File::create(rust_filename)?;
 
-                write!(file, "{notice}")?;
+                let cfg = AARCH_CONFIGURATIONS;
+                let definitions = F16_FORMATTING_DEF;
+                write_lib_rs(&mut file, architecture, notice, cfg, definitions, chunk)?;
 
-                writeln!(file, "use core_arch::arch::{architecture}::*;")?;
-                writeln!(file, "use crate::{{debug_simd_finish, debug_f16}};")?;
+                let toml_filename = format!("rust_programs/mod_{i}/Cargo.toml");
+                trace!("generating `{toml_filename}`");
+                let mut file = File::create(toml_filename).unwrap();
 
-                for intrinsic in chunk {
-                    crate::common::gen_rust::create_rust_test_module(&mut file, intrinsic)?;
-                }
+                write_lib_cargo_toml(&mut file, &format!("mod_{i}"))?;
 
                 Ok(())
             })

crates/intrinsic-test/src/common/compare.rs

@@ -24,6 +24,7 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
             let rust = runner_command(runner)
                 .arg(format!("target/{target}/release/intrinsic-test-programs"))
                 .arg(intrinsic_name)
+                .current_dir("rust_programs")
                 .output();
 
             let (c, rust) = match (c, rust) {

crates/intrinsic-test/src/common/gen_rust.rs

@@ -9,46 +9,53 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition;
 // The number of times each intrinsic will be called.
 const PASSES: u32 = 20;
 
-pub fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> {
+fn write_cargo_toml_header(w: &mut impl std::io::Write, name: &str) -> std::io::Result<()> {
     writeln!(
         w,
         concat!(
             "[package]\n",
-            "name = \"intrinsic-test-programs\"\n",
+            "name = \"{name}\"\n",
             "version = \"{version}\"\n",
             "authors = [{authors}]\n",
             "license = \"{license}\"\n",
             "edition = \"2018\"\n",
-            "[workspace]\n",
-            "[dependencies]\n",
-            "core_arch = {{ path = \"../crates/core_arch\" }}",
         ),
+        name = name,
         version = env!("CARGO_PKG_VERSION"),
         authors = env!("CARGO_PKG_AUTHORS")
             .split(":")
             .format_with(", ", |author, fmt| fmt(&format_args!("\"{author}\""))),
         license = env!("CARGO_PKG_LICENSE"),
-    )?;
+    )
+}
 
-    for binary in binaries {
-        writeln!(
-            w,
-            concat!(
-                "[[bin]]\n",
-                "name = \"{binary}\"\n",
-                "path = \"{binary}/main.rs\"\n",
-            ),
-            binary = binary,
-        )?;
+pub fn write_bin_cargo_toml(
+    w: &mut impl std::io::Write,
+    module_count: usize,
+) -> std::io::Result<()> {
+    write_cargo_toml_header(w, "intrinsic-test-programs")?;
+
+    writeln!(w, "[dependencies]")?;
+
+    for i in 0..module_count {
+        writeln!(w, "mod_{i} = {{ path = \"mod_{i}/\" }}")?;
     }
 
     Ok(())
 }
 
+pub fn write_lib_cargo_toml(w: &mut impl std::io::Write, name: &str) -> std::io::Result<()> {
+    write_cargo_toml_header(w, name)?;
+
+    writeln!(w, "[dependencies]")?;
+    writeln!(w, "core_arch = {{ path = \"../../crates/core_arch\" }}")?;
+
+    Ok(())
+}
+
 pub fn write_main_rs<'a>(
     w: &mut impl std::io::Write,
-    available_parallelism: usize,
-    architecture: &str,
+    chunk_count: usize,
     cfg: &str,
     definitions: &str,
     intrinsics: impl Iterator<Item = &'a str> + Clone,
@@ -65,10 +72,7 @@ pub fn write_main_rs<'a>(
     writeln!(w, "{cfg}")?;
     writeln!(w, "{definitions}")?;
 
-    writeln!(w, "use core_arch::arch::{architecture}::*;")?;
-
-    for module in 0..Ord::min(available_parallelism, intrinsics.clone().count()) {
-        writeln!(w, "mod mod_{module};")?;
+    for module in 0..chunk_count {
         writeln!(w, "use mod_{module}::*;")?;
     }
 
@@ -91,6 +95,38 @@ pub fn write_main_rs<'a>(
     Ok(())
 }
 
+pub fn write_lib_rs<T: IntrinsicTypeDefinition>(
+    w: &mut impl std::io::Write,
+    architecture: &str,
+    notice: &str,
+    cfg: &str,
+    definitions: &str,
+    intrinsics: &[impl IntrinsicDefinition<T>],
+) -> std::io::Result<()> {
+    write!(w, "{notice}")?;
+
+    writeln!(w, "#![feature(simd_ffi)]")?;
+    writeln!(w, "#![feature(f16)]")?;
+    writeln!(w, "#![allow(unused)]")?;
+
+    // Cargo will spam the logs if these warnings are not silenced.
+    writeln!(w, "#![allow(non_upper_case_globals)]")?;
+    writeln!(w, "#![allow(non_camel_case_types)]")?;
+    writeln!(w, "#![allow(non_snake_case)]")?;
+
+    writeln!(w, "{cfg}")?;
+
+    writeln!(w, "use core_arch::arch::{architecture}::*;")?;
+
+    writeln!(w, "{definitions}")?;
+
+    for intrinsic in intrinsics {
+        crate::common::gen_rust::create_rust_test_module(w, intrinsic)?;
+    }
+
+    Ok(())
+}
+
 pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Option<&str>) -> bool {
     /* If there has been a linker explicitly set from the command line then
      * we want to set it via setting it in the RUSTFLAGS*/
@@ -100,6 +136,9 @@ pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Opti
     let mut cargo_command = Command::new("cargo");
     cargo_command.current_dir("rust_programs");
 
+    // Do not use the target directory of the workspace please.
+    cargo_command.env("CARGO_TARGET_DIR", "target");
+
     if let Some(toolchain) = toolchain
         && !toolchain.is_empty()
     {