Rust: add proc-macro capabilities to QL tests

This adds the possibility to add a special `proc_macro.rs` source file
to QL tests, which will be generated into a `proc_macro` crate the
usual `lib` crate depends on.

This allow to define procedural macros in QL tests, and is here used to
move the `macro-expansion` integration test to be a language test
instead.

As the generated manifests involved were starting to get a bit complex,
they are now generated from a `mustache` template.

Author: redsun82

Cargo.lock

@@ -7,6 +7,7 @@ codeql_rust_binary(
     name = "extractor",
     srcs = glob(["src/**/*.rs"]),
     aliases = aliases(),
+    compile_data = ["src/qltest_cargo.mustache"],
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
     ) + [

misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl

@@ -41,3 +41,4 @@ tracing = "0.1.41"
 tracing-flame = "0.2.0"
 tracing-subscriber = "0.3.19"
 chalk-ir = "0.100.0"
+mustache = "0.9.0"

rust/extractor/BUILD.bazel

@@ -16,39 +16,54 @@ fn dump_lib() -> anyhow::Result<()> {
     let lib = paths
         .iter()
         .map(|p| p.file_stem().expect("results of glob must have a name"))
-        .filter(|&p| !["main", "lib"].map(OsStr::new).contains(&p))
+        .filter(|&p| !["main", "lib", "proc_macro"].map(OsStr::new).contains(&p))
         .map(|p| format!("mod {};", p.to_string_lossy()))
         .join("\n");
     fs::write("lib.rs", lib).context("writing lib.rs")
 }
 
-fn dump_cargo_manifest(dependencies: &[String]) -> anyhow::Result<()> {
-    let mut manifest = String::from(
-        r#"[workspace]
-[package]
-name = "test"
-version="0.0.1"
-edition="2021"
-[lib]
-path="lib.rs"
-"#,
-    );
-    if fs::exists("main.rs").context("checking existence of main.rs")? {
-        manifest.push_str(
-            r#"[[bin]]
-name = "main"
-path = "main.rs"
-"#,
-        );
+#[derive(serde::Serialize)]
+enum TestCargoManifest<'a> {
+    Workspace {},
+    Lib {
+        uses_proc_macro: bool,
+        uses_main: bool,
+        dependencies: &'a [String],
+    },
+    Macro {},
+}
+
+impl TestCargoManifest<'_> {
+    pub fn dump(&self, path: impl AsRef<Path>) -> anyhow::Result<()> {
+        static TEMPLATE: std::sync::LazyLock<mustache::Template> = std::sync::LazyLock::new(|| {
+            mustache::compile_str(include_str!("qltest_cargo.mustache"))
+                .expect("compiling template")
+        });
+
+        let path = path.as_ref();
+        fs::create_dir_all(path).with_context(|| format!("creating {}", path.display()))?;
+        let path = path.join("Cargo.toml");
+        let rendered = TEMPLATE
+            .render_to_string(&self)
+            .with_context(|| format!("rendering {}", path.display()))?;
+        fs::write(&path, rendered).with_context(|| format!("writing {}", path.display()))
     }
-    if !dependencies.is_empty() {
-        manifest.push_str("[dependencies]\n");
-        for dep in dependencies {
-            manifest.push_str(dep);
-            manifest.push('\n');
-        }
+}
+fn dump_cargo_manifest(dependencies: &[String]) -> anyhow::Result<()> {
+    let uses_proc_macro =
+        fs::exists("proc_macro.rs").context("checking existence of proc_macro.rs")?;
+    let lib_manifest = TestCargoManifest::Lib {
+        uses_proc_macro,
+        uses_main: fs::exists("main.rs").context("checking existence of main.rs")?,
+        dependencies,
+    };
+    if uses_proc_macro {
+        TestCargoManifest::Workspace {}.dump("")?;
+        lib_manifest.dump(".lib")?;
+        TestCargoManifest::Macro {}.dump(".proc_macro")
+    } else {
+        lib_manifest.dump("")
     }
-    fs::write("Cargo.toml", manifest).context("writing Cargo.toml")
 }
 
 fn set_sources(config: &mut Config) -> anyhow::Result<()> {

rust/extractor/Cargo.toml

@@ -0,0 +1,42 @@
+{{#Workspace}}
+[workspace]
+resolver = "2"
+members = [".lib", ".proc_macro"]
+{{/Workspace}}
+
+{{#Lib}}
+{{^uses_proc_macro}}
+[workspace]
+{{/uses_proc_macro}}
+[package]
+name = "test"
+version = "0.1.0"
+edition = "2021"
+[lib]
+path = "{{#uses_proc_macro}}../{{/uses_proc_macro}}lib.rs"
+{{#has_main}}
+[[bin]]
+name = "main"
+path = "{{#uses_proc_macro}}../{{/uses_proc_macro}}main.rs"
+{{/has_main}}
+[dependencies]
+{{#uses_proc_macro}}
+proc_macro = { path = "../.proc_macro" }
+{{/uses_proc_macro}}
+{{#dependencies}}
+{{.}}
+{{/dependencies}}
+{{/Lib}}
+
+{{#Macro}}
+[package]
+name = "proc_macro"
+version = "0.1.0"
+edition = "2021"
+[lib]
+path = "../proc_macro.rs"
+proc_macro = true
+[dependencies]
+quote = "1.0.40"
+syn = { version = "2.0.100", features = ["full"] }
+{{/Macro}}

rust/extractor/src/qltest.rs

@@ -1,3 +1,6 @@
 Cargo.toml
 lib.rs
 target/
+.proc_macro/
+.lib/
+

rust/extractor/src/qltest_cargo.mustache

@@ -0,0 +1,3 @@
+multiplePathResolutions
+| macro_expansion.rs:1:5:1:14 | proc_macro | file://:0:0:0:0 | Crate([email protected]) |
+| macro_expansion.rs:1:5:1:14 | proc_macro | proc_macro.rs:0:0:0:0 | Crate([email protected]) |