From 9c284b1778e93a396085c2fe07527e91784dc5e5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 4 Jul 2025 12:22:23 +0200 Subject: [PATCH 01/23] Just: introduce scaffolding for common verbs, and apply to rust --- justfile | 2 + misc/bazel/justfile | 5 +++ misc/just/codeql-test-run.ts | 39 ++++++++++++++++++ misc/just/forward-command.ts | 64 ++++++++++++++++++++++++++++++ misc/just/forward.just | 28 +++++++++++++ misc/just/lib.just | 63 +++++++++++++++++++++++++++++ misc/just/semmle-code-stub.just | 1 + rust/justfile | 11 +++++ rust/lint.py | 48 ++++++++++++++-------- rust/ql/integration-tests/justfile | 9 +++++ rust/ql/justfile | 4 ++ rust/ql/test/justfile | 17 ++++++++ 12 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 justfile create mode 100644 misc/bazel/justfile create mode 100644 misc/just/codeql-test-run.ts create mode 100644 misc/just/forward-command.ts create mode 100644 misc/just/forward.just create mode 100644 misc/just/lib.just create mode 100644 misc/just/semmle-code-stub.just create mode 100644 rust/justfile create mode 100644 rust/ql/integration-tests/justfile create mode 100644 rust/ql/justfile create mode 100644 rust/ql/test/justfile diff --git a/justfile b/justfile new file mode 100644 index 000000000000..a25d51fd9a5d --- /dev/null +++ b/justfile @@ -0,0 +1,2 @@ +import 'misc/just/lib.just' +import 'misc/just/forward.just' diff --git a/misc/bazel/justfile b/misc/bazel/justfile new file mode 100644 index 000000000000..55b2eb9dbd8e --- /dev/null +++ b/misc/bazel/justfile @@ -0,0 +1,5 @@ +import '../just/lib.just' + +[no-cd, positional-arguments, no-exit-message] +hello +ARGS: + @echo "hello from bzl" "$@" diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts new file mode 100644 index 000000000000..ae68ba9dde50 --- /dev/null +++ b/misc/just/codeql-test-run.ts @@ -0,0 +1,39 @@ +import * as child_process from "child_process"; +import * as path from "path"; + +function codeqlTestRun(argv: string[]): number { + const [language, args, ...plus] = argv; + let codeql = + process.env["SEMMLE_CODE"] ? + path.join(process.env["SEMMLE_CODE"], "target", "intree", `codeql-${language}`, "codeql") + : + "codeql" + ; + process.env["CODEQL_CONFIG_FILE"] ||= "." // disable the default implicit config file, but keep an explicit one + let plus_options = plus.map(option => option.trim().split("\n").filter(option => option !== "")); + let testing_level = 0; + let parsed_args = args.split(" ").filter(arg => { + if (arg === "") return false; + if (/^\++$/.test(arg)) { + testing_level = Math.max(testing_level, arg.length); + return false; + } + return true; + }); + if (parsed_args.every(arg => arg.startsWith("-"))) { + parsed_args.push("."); + } + let invocation = [codeql, "test", "run", "-j0", ...parsed_args]; + for (let i = 0; i < Math.min(plus_options.length, testing_level); i++) { + invocation.push(...plus_options[i]); + } + console.log(`${process.env["CMD_BEGIN"] || ""}${invocation.join(" ")}${process.env["CMD_END"] || ""}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit" }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(codeqlTestRun(process.argv.slice(2))); diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts new file mode 100644 index 000000000000..92078b065583 --- /dev/null +++ b/misc/just/forward-command.ts @@ -0,0 +1,64 @@ +import * as child_process from "child_process"; +import * as path from "path"; +import * as fs from "fs"; + +function commonDir(paths: string[]): string { + if (paths.length === 0) return ""; + const splitPaths = paths.map(p => p.split(path.sep)); + let i; + for (i = 0; i < splitPaths[0].length; i++) { + if (!splitPaths.every(parts => parts[i] === splitPaths[0][i])) { + break; + } + } + const commonParts = splitPaths[0].slice(0, i); + let ret = commonParts.join(path.sep); + if (!fs.existsSync(ret)) { + throw new Error(`Common directory does not exist: ${ret}`); + } + if (!fs.lstatSync(ret).isDirectory()) { + ret = path.dirname(ret); + } + return ret; +} + +function forwardCommand(args: string[]): number { + // Avoid infinite recursion + if (args.length == 0) { + console.error("No command provided"); + return 1; + } + const cmd = args[0]; + const envVariable = `__JUST_FORWARD_${cmd}`; + if (process.env[envVariable]) { + console.error(`No common ${cmd} handler found`); + return 1; + } + process.env[envVariable] = "true"; + const cmdArgs = args.slice(1); + const is_flag = /^(-.*|\++)$/; // + is used for testing level in some langauge tests + const flags = cmdArgs.filter(arg => is_flag.test(arg)); + const positionalArgs = cmdArgs.filter(arg => !is_flag.test(arg)); + + if (positionalArgs.length === 0) { + console.error("No positional arguments provided"); + return 1; + } + + const commonPath = commonDir(positionalArgs); + let relativeArgs = positionalArgs.map(arg => path.relative(commonPath, arg) || "."); + if (relativeArgs.length === 1 && relativeArgs[0] === ".") { + relativeArgs = []; + } + + const invocation = [process.env["JUST_EXECUTABLE"] || "just", cmd, ...flags, ...relativeArgs]; + console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: commonPath }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(forwardCommand(process.argv.slice(2))); diff --git a/misc/just/forward.just b/misc/just/forward.just new file mode 100644 index 000000000000..fc3b1fde187f --- /dev/null +++ b/misc/just/forward.just @@ -0,0 +1,28 @@ +import "lib.just" + +# copy&paste necessary for each command until proper forwarding of multiple args is implemented +# see https://github.com/casey/just/issues/1988 + +_forward := tsx + source_dir() + "/forward-command.ts" + + +[no-cd, positional-arguments, no-exit-message] +@test *ARGS=".": + {{ _forward }} test "$@" + + +[no-cd, positional-arguments, no-exit-message] +@build *ARGS=".": + {{ _forward }} build "$@" + +[no-cd, positional-arguments, no-exit-message] +@generate *ARGS=".": + {{ _forward }} generate "$@" + +[no-cd, positional-arguments, no-exit-message] +@lint *ARGS=".": + {{ _forward }} lint "$@" + +[no-cd, positional-arguments, no-exit-message] +@format *ARGS=".": + {{ _forward }} format "$@" diff --git a/misc/just/lib.just b/misc/just/lib.just new file mode 100644 index 000000000000..8a6dc68f895c --- /dev/null +++ b/misc/just/lib.just @@ -0,0 +1,63 @@ +set fallback +set allow-duplicate-recipes +set allow-duplicate-variables +set unstable + +export PATH_SEP := if os() == "windows" { ";" } else { ":" } +export JUST_EXECUTABLE := just_executable() + +error := style("error") + "error" + NORMAL + ": " +cmd_sep := "\n#--------------------------------------------------------\n" +export CMD_BEGIN := style("command") + cmd_sep +export CMD_END := cmd_sep + NORMAL + +tsx := "npx tsx@4.19.0 " + +import? '../../../semmle-code.just' # internal repo just file, if present +import 'semmle-code-stub.just' + + +[no-exit-message] +@_require_semmle_code: + {{ if SEMMLE_CODE == "" { ''' + echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2 + exit 1 + ''' } else { "" } }} + +_build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE) + +[no-exit-message] +_maybe_build LANGUAGE: + {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd $SEMMLE_CODE; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} + +[no-cd, positional-arguments, no-exit-message] +@_codeql_test LANGUAGE +ARGS: (_maybe_build LANGUAGE) + {{ tsx + source_dir() }}/codeql-test-run.ts "$@" + +[no-cd, positional-arguments, no-exit-message] +@_codeql_test_run_only LANGUAGE +ARGS: + {{ tsx + source_dir() }}/codeql-test-run.ts "$@" + + +[no-cd, no-exit-message] +_ql_format +ARGS: (_maybe_build "nolang") + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/target/intree/codeql-nolang/' } else { '' } }}codeql query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} + + +[no-cd, no-exit-message] +_bazel COMMAND *ARGS: + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/tools/' } else { '' } }}bazel {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} + +[no-cd, no-exit-message] +_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS) + +[no-cd] +_run +ARGS: + {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }} + +[no-cd, positional-arguments, no-exit-message] +_just +ARGS: + {{ JUST_EXECUTABLE }} "$@" + +[no-cd] +_black *ARGS=".": (_run "uv" "run" "black" ARGS) diff --git a/misc/just/semmle-code-stub.just b/misc/just/semmle-code-stub.just new file mode 100644 index 000000000000..14733ffb648e --- /dev/null +++ b/misc/just/semmle-code-stub.just @@ -0,0 +1 @@ +export SEMMLE_CODE := "" diff --git a/rust/justfile b/rust/justfile new file mode 100644 index 000000000000..bcd753087da7 --- /dev/null +++ b/rust/justfile @@ -0,0 +1,11 @@ +import '../misc/just/lib.just' + +install: (_bazel "run" "@codeql//rust:install") + +build: generate (_build "rust") + +generate: (_bazel "run" "@codeql//rust/codegen") + +lint: (_run "python3" "lint.py") + +format: (_run "python3" "lint.py" "--format-only") diff --git a/rust/lint.py b/rust/lint.py index 600a888649e9..e3a078635fa5 100755 --- a/rust/lint.py +++ b/rust/lint.py @@ -4,6 +4,14 @@ import pathlib import shutil import sys +import argparse + + +def options(): + parser = argparse.ArgumentParser(description="lint rust language pack code") + parser.add_argument("--format-only", action="store_true", help="Only apply formatting") + return parser.parse_args() + def tool(name): @@ -12,27 +20,35 @@ def tool(name): return ret -this_dir = pathlib.Path(__file__).resolve().parent +def main(): + args = options() + this_dir = pathlib.Path(__file__).resolve().parent + + + cargo = tool("cargo") + bazel = tool("bazel") + + runs = [] -cargo = tool("cargo") -bazel = tool("bazel") -runs = [] + def run(tool, args, *, cwd=this_dir): + print("+", tool, args) + runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) -def run(tool, args, *, cwd=this_dir): - print("+", tool, args) - runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) + # make sure bazel-provided sources are put in tree for `cargo` to work with them + run(bazel, "run ast-generator:inject-sources") + run(cargo, "fmt --all --quiet") + if not args.format_only: + for manifest in this_dir.rglob("Cargo.toml"): + if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): + run(cargo, + "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", + cwd=manifest.parent) -# make sure bazel-provided sources are put in tree for `cargo` to work with them -run(bazel, "run ast-generator:inject-sources") -run(cargo, "fmt --all --quiet") + return max(r.returncode for r in runs) -for manifest in this_dir.rglob("Cargo.toml"): - if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): - run(cargo, - "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", - cwd=manifest.parent) -sys.exit(max(r.returncode for r in runs)) +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/ql/integration-tests/justfile b/rust/ql/integration-tests/justfile new file mode 100644 index 000000000000..c8fad074a3f9 --- /dev/null +++ b/rust/ql/integration-tests/justfile @@ -0,0 +1,9 @@ +import "../../../misc/just/lib.just" + + +[no-cd] +test *ARGS=".": (_just "generate") (_integration_test ARGS) + +# TODO in separate PR +# [no-cd] +# format *ARGS=".": (_ql_format ARGS) (_black ARGS) diff --git a/rust/ql/justfile b/rust/ql/justfile new file mode 100644 index 000000000000..d4f2aa8fc826 --- /dev/null +++ b/rust/ql/justfile @@ -0,0 +1,4 @@ +import "../../misc/just/lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) diff --git a/rust/ql/test/justfile b/rust/ql/test/justfile new file mode 100644 index 000000000000..c550af68a14e --- /dev/null +++ b/rust/ql/test/justfile @@ -0,0 +1,17 @@ +import '../../../misc/just/lib.just' + +plus := """ + --check-databases + --check-diff-informed + --fail-on-trap-errors + --check-undefined-labels + --check-unused-labels + --check-repeated-labels + --check-redefined-labels + --check-use-before-definition +""" + +plusplus := "--consistency-queries=" + source_dir() + "/../consistency-queries" + +[no-cd] +test *ARGS=".": (_codeql_test "rust" ARGS plus plusplus) \ No newline at end of file From 1202af1c5c44d5ce25c7f16d7100edf611fc0a11 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 4 Jul 2025 13:45:10 +0200 Subject: [PATCH 02/23] Just: fix for windows --- misc/just/forward.just | 2 +- misc/just/lib.just | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/misc/just/forward.just b/misc/just/forward.just index fc3b1fde187f..64af38d272fa 100644 --- a/misc/just/forward.just +++ b/misc/just/forward.just @@ -3,7 +3,7 @@ import "lib.just" # copy&paste necessary for each command until proper forwarding of multiple args is implemented # see https://github.com/casey/just/issues/1988 -_forward := tsx + source_dir() + "/forward-command.ts" +_forward := tsx + ' "' + source_dir() + '/forward-command.ts"' [no-cd, positional-arguments, no-exit-message] diff --git a/misc/just/lib.just b/misc/just/lib.just index 8a6dc68f895c..563ffa1137fa 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -11,7 +11,7 @@ cmd_sep := "\n#--------------------------------------------------------\n" export CMD_BEGIN := style("command") + cmd_sep export CMD_END := cmd_sep + NORMAL -tsx := "npx tsx@4.19.0 " +tsx := "npx tsx@4.19.0" import? '../../../semmle-code.just' # internal repo just file, if present import 'semmle-code-stub.just' @@ -32,11 +32,11 @@ _maybe_build LANGUAGE: [no-cd, positional-arguments, no-exit-message] @_codeql_test LANGUAGE +ARGS: (_maybe_build LANGUAGE) - {{ tsx + source_dir() }}/codeql-test-run.ts "$@" + {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" [no-cd, positional-arguments, no-exit-message] @_codeql_test_run_only LANGUAGE +ARGS: - {{ tsx + source_dir() }}/codeql-test-run.ts "$@" + {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" [no-cd, no-exit-message] From 2dea9da38cd2dc8aed017c9dff9aa739a89d6921 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 4 Jul 2025 13:45:31 +0200 Subject: [PATCH 03/23] Just: add `codegen` --- misc/codegen/justfile | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 misc/codegen/justfile diff --git a/misc/codegen/justfile b/misc/codegen/justfile new file mode 100644 index 000000000000..9dd2019b5fac --- /dev/null +++ b/misc/codegen/justfile @@ -0,0 +1,5 @@ +import "../just/lib.just" + +test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...") + +format *ARGS=".": (_black ARGS) \ No newline at end of file From 9e31fb50c8d5d508f7776b2742d71c14283f71eb Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 4 Jul 2025 16:00:28 +0200 Subject: [PATCH 04/23] Just: fix and add windows --- misc/just/codeql-test-run.ts | 104 ++++++++++++++++++++++------ misc/just/forward-command.ts | 7 +- misc/just/lib.just | 27 +++++--- rust/ql/justfile | 7 ++ rust/ql/test/justfile | 17 ----- swift/justfile | 15 ++++ swift/ql/integration-tests/justfile | 9 +++ swift/ql/justfile | 16 +++++ 8 files changed, 151 insertions(+), 51 deletions(-) delete mode 100644 rust/ql/test/justfile create mode 100644 swift/justfile create mode 100644 swift/ql/integration-tests/justfile create mode 100644 swift/ql/justfile diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts index ae68ba9dde50..ab3bcee37d60 100644 --- a/misc/just/codeql-test-run.ts +++ b/misc/just/codeql-test-run.ts @@ -1,39 +1,97 @@ import * as child_process from "child_process"; import * as path from "path"; +import * as os from "os"; + + + +function invoke(invocation: string[], options: {cwd?: string, log_prefix?: string} = {}) : number { + const log_prefix = options.log_prefix && options.log_prefix !== "" ? `${options.log_prefix} ` : ""; + console.log(`${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: options.cwd }); + } catch (error) { + return 1; + } + return 0; +} + +type Args = { + tests: string[]; + flags: string[]; + env: string[]; + build: boolean; + testing_level: number; +}; + +function parseArgs(args: Args, argv: string) { + argv + .split(/(? { + if (arg === "--no-build") { + args.build = false; + } else if (arg.startsWith("-")) { + args.flags.push(arg); + } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { + args.env.push(arg); + } else if (/^\++$/.test(arg)) { + args.testing_level = Math.max(args.testing_level, arg.length); + } else if (arg !== "") { + args.tests.push(arg); + } + }); +} + function codeqlTestRun(argv: string[]): number { - const [language, args, ...plus] = argv; + const [language, extra_args, ...plus] = argv; let codeql = process.env["SEMMLE_CODE"] ? path.join(process.env["SEMMLE_CODE"], "target", "intree", `codeql-${language}`, "codeql") : "codeql" ; - process.env["CODEQL_CONFIG_FILE"] ||= "." // disable the default implicit config file, but keep an explicit one - let plus_options = plus.map(option => option.trim().split("\n").filter(option => option !== "")); - let testing_level = 0; - let parsed_args = args.split(" ").filter(arg => { - if (arg === "") return false; - if (/^\++$/.test(arg)) { - testing_level = Math.max(testing_level, arg.length); - return false; - } - return true; - }); - if (parsed_args.every(arg => arg.startsWith("-"))) { - parsed_args.push("."); + const ram_per_thread = process.platform === "linux" ? 3000 : 2048; + const cpus = os.cpus().length; + let args: Args = { + tests: [], + flags: [ + `--ram=${ram_per_thread}`, + `-j${cpus}`, + ], + env: [], + build: true, + testing_level: 0 + }; + parseArgs(args, extra_args); + for (let i = 0; i < Math.min(plus.length, args.testing_level); i++) { + parseArgs(args, plus[i]); } - let invocation = [codeql, "test", "run", "-j0", ...parsed_args]; - for (let i = 0; i < Math.min(plus_options.length, testing_level); i++) { - invocation.push(...plus_options[i]); + if (args.tests.length === 0) { + args.tests.push("."); } - console.log(`${process.env["CMD_BEGIN"] || ""}${invocation.join(" ")}${process.env["CMD_END"] || ""}`); - try { - child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit" }); - } catch (error) { - return 1; + if (args.build && process.env["SEMMLE_CODE"]) { + // If SEMMLE_CODE is set, we are in the semmle-code repo, so we build the codeql binary. + // Otherwise, we use codeql from PATH. + if (invoke(["python3", "build", `target/intree/codeql-${language}`], {cwd: process.env["SEMMLE_CODE"]}) !== 0) { + return 1; + } } - return 0; + process.env["CODEQL_CONFIG_FILE"] ||= "." // disable the default implicit config file, but keep an explicit one + // Set and unset environment variables + args.env.forEach((envVar) => { + const [key, value] = envVar.split("=", 2); + if (key) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } else { + console.error(`Invalid environment variable assignment: ${envVar}`); + process.exit(1); + } + }); + return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {log_prefix: args.env.join(" ")}); } process.exit(codeqlTestRun(process.argv.slice(2))); diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index 92078b065583..65e834a98425 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -36,9 +36,10 @@ function forwardCommand(args: string[]): number { } process.env[envVariable] = "true"; const cmdArgs = args.slice(1); - const is_flag = /^(-.*|\++)$/; // + is used for testing level in some langauge tests - const flags = cmdArgs.filter(arg => is_flag.test(arg)); - const positionalArgs = cmdArgs.filter(arg => !is_flag.test(arg)); + // non-positional arguments are flags, repeated + (used by language tests) or environment variable settings + const is_non_positional = /^(-.*|\++|[A-Z_][A-Z_0-9]*=.*)$/; + const flags = cmdArgs.filter(arg => is_non_positional.test(arg)); + const positionalArgs = cmdArgs.filter(arg => !is_non_positional.test(arg)); if (positionalArgs.length === 0) { console.error("No positional arguments provided"); diff --git a/misc/just/lib.just b/misc/just/lib.just index 563ffa1137fa..eb16bd5a596d 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -28,25 +28,31 @@ _build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE) [no-exit-message] _maybe_build LANGUAGE: - {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd $SEMMLE_CODE; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} + {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} -[no-cd, positional-arguments, no-exit-message] -@_codeql_test LANGUAGE +ARGS: (_maybe_build LANGUAGE) - {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" + +default_db_checks := """\ + --check-databases \ + --check-diff-informed \ + --fail-on-trap-errors \ +""" [no-cd, positional-arguments, no-exit-message] -@_codeql_test_run_only LANGUAGE +ARGS: +@_codeql_test LANGUAGE +ARGS: {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" [no-cd, no-exit-message] _ql_format +ARGS: (_maybe_build "nolang") - {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/target/intree/codeql-nolang/' } else { '' } }}codeql query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} [no-cd, no-exit-message] _bazel COMMAND *ARGS: - {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/tools/' } else { '' } }}bazel {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/tools/bazel"' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} + +[no-cd, no-exit-message] +_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS) [no-cd, no-exit-message] _integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS) @@ -55,9 +61,14 @@ _integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" _run +ARGS: {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }} +[no-cd] +_run_in_semmle_code +ARGS: _require_semmle_code + {{ cmd_sep }}cd "$SEMMLE_CODE"; {{ ARGS }}{{ cmd_sep }} + + [no-cd, positional-arguments, no-exit-message] _just +ARGS: - {{ JUST_EXECUTABLE }} "$@" + "{{ JUST_EXECUTABLE }}" "$@" [no-cd] _black *ARGS=".": (_run "uv" "run" "black" ARGS) diff --git a/rust/ql/justfile b/rust/ql/justfile index d4f2aa8fc826..a01e673cdefd 100644 --- a/rust/ql/justfile +++ b/rust/ql/justfile @@ -2,3 +2,10 @@ import "../../misc/just/lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) + +db_checks := default_db_checks + +consistency := "--consistency-queries=" + source_dir() / "consistency-queries" + +[no-cd] +test *ARGS=".": (_codeql_test "rust" ARGS db_checks consistency) \ No newline at end of file diff --git a/rust/ql/test/justfile b/rust/ql/test/justfile deleted file mode 100644 index c550af68a14e..000000000000 --- a/rust/ql/test/justfile +++ /dev/null @@ -1,17 +0,0 @@ -import '../../../misc/just/lib.just' - -plus := """ - --check-databases - --check-diff-informed - --fail-on-trap-errors - --check-undefined-labels - --check-unused-labels - --check-repeated-labels - --check-redefined-labels - --check-use-before-definition -""" - -plusplus := "--consistency-queries=" + source_dir() + "/../consistency-queries" - -[no-cd] -test *ARGS=".": (_codeql_test "rust" ARGS plus plusplus) \ No newline at end of file diff --git a/swift/justfile b/swift/justfile new file mode 100644 index 000000000000..d63d7f32e4fd --- /dev/null +++ b/swift/justfile @@ -0,0 +1,15 @@ +import '../misc/just/lib.just' + +install: (_bazel "run" "@codeql//swift:install") + +build: (_build "swift") + +generate: (_bazel "run" "@codeql//swift/codegen") + +@_check_clang_format: + if ! which clang-format > /dev/null; then \ + "{{ JUST_EXECUTABLE }}" _run_in_semmle_code "tools/bazel" "run" "//c/clang-format:install"; \ + fi + +format ARGS=".": _check_clang_format (_run "clang-format" "-i" ("$(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')")) + diff --git a/swift/ql/integration-tests/justfile b/swift/ql/integration-tests/justfile new file mode 100644 index 000000000000..c8fad074a3f9 --- /dev/null +++ b/swift/ql/integration-tests/justfile @@ -0,0 +1,9 @@ +import "../../../misc/just/lib.just" + + +[no-cd] +test *ARGS=".": (_just "generate") (_integration_test ARGS) + +# TODO in separate PR +# [no-cd] +# format *ARGS=".": (_ql_format ARGS) (_black ARGS) diff --git a/swift/ql/justfile b/swift/ql/justfile new file mode 100644 index 000000000000..295853511477 --- /dev/null +++ b/swift/ql/justfile @@ -0,0 +1,16 @@ +import "../../misc/just/lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +db_checks := default_db_checks + """\ + --check-repeated-labels \ + --check-redefined-labels \ + --check-use-before-definition \ + --consistency-queries=ql/swift/ql/consistency-queries \ +""" + +consistency := "--consistency-queries=" + source_dir() / "consistency-queries" + +[no-cd] +test *ARGS=".": (_codeql_test "swift" ARGS db_checks consistency) \ No newline at end of file From 5b9436a95f785703dfa6825e8668f5839dd6b6d8 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 4 Jul 2025 17:27:43 +0200 Subject: [PATCH 05/23] Just: fix swift tests --- swift/ql/justfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swift/ql/justfile b/swift/ql/justfile index 295853511477..37d2d4601ca6 100644 --- a/swift/ql/justfile +++ b/swift/ql/justfile @@ -7,10 +7,9 @@ db_checks := default_db_checks + """\ --check-repeated-labels \ --check-redefined-labels \ --check-use-before-definition \ - --consistency-queries=ql/swift/ql/consistency-queries \ """ consistency := "--consistency-queries=" + source_dir() / "consistency-queries" [no-cd] -test *ARGS=".": (_codeql_test "swift" ARGS db_checks consistency) \ No newline at end of file +test *ARGS=".": (_codeql_test "swift" ARGS db_checks consistency) From a4acf0890e22c96dc546f807910c747928da9e58 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 7 Jul 2025 15:52:20 +0200 Subject: [PATCH 06/23] Just: fix ram option for `codeql test run` --- misc/just/codeql-test-run.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts index ab3bcee37d60..42c9f8ede4ad 100644 --- a/misc/just/codeql-test-run.ts +++ b/misc/just/codeql-test-run.ts @@ -12,7 +12,7 @@ function invoke(invocation: string[], options: {cwd?: string, log_prefix?: strin } catch (error) { return 1; } - return 0; + return 0; } type Args = { @@ -55,7 +55,7 @@ function codeqlTestRun(argv: string[]): number { let args: Args = { tests: [], flags: [ - `--ram=${ram_per_thread}`, + `--ram=${ram_per_thread * cpus}`, `-j${cpus}`, ], env: [], From d7d7cf920a8916329fce1c5962680a14c705de25 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 7 Jul 2025 15:52:53 +0200 Subject: [PATCH 07/23] Just: format `ts` files --- misc/just/codeql-test-run.ts | 84 ++++++++++++++------------ misc/just/forward-command.ts | 110 +++++++++++++++++++---------------- 2 files changed, 109 insertions(+), 85 deletions(-) diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts index 42c9f8ede4ad..746066ecddda 100644 --- a/misc/just/codeql-test-run.ts +++ b/misc/just/codeql-test-run.ts @@ -2,13 +2,22 @@ import * as child_process from "child_process"; import * as path from "path"; import * as os from "os"; - - -function invoke(invocation: string[], options: {cwd?: string, log_prefix?: string} = {}) : number { - const log_prefix = options.log_prefix && options.log_prefix !== "" ? `${options.log_prefix} ` : ""; - console.log(`${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`); +function invoke( + invocation: string[], + options: { cwd?: string; log_prefix?: string } = {}, +): number { + const log_prefix = + options.log_prefix && options.log_prefix !== "" + ? `${options.log_prefix} ` + : ""; + console.log( + `${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`, + ); try { - child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: options.cwd }); + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd: options.cwd, + }); } catch (error) { return 1; } @@ -24,43 +33,40 @@ type Args = { }; function parseArgs(args: Args, argv: string) { - argv - .split(/(? { - if (arg === "--no-build") { - args.build = false; - } else if (arg.startsWith("-")) { - args.flags.push(arg); - } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { - args.env.push(arg); - } else if (/^\++$/.test(arg)) { - args.testing_level = Math.max(args.testing_level, arg.length); - } else if (arg !== "") { - args.tests.push(arg); - } - }); + argv.split(/(? { + if (arg === "--no-build") { + args.build = false; + } else if (arg.startsWith("-")) { + args.flags.push(arg); + } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { + args.env.push(arg); + } else if (/^\++$/.test(arg)) { + args.testing_level = Math.max(args.testing_level, arg.length); + } else if (arg !== "") { + args.tests.push(arg); + } + }); } - function codeqlTestRun(argv: string[]): number { const [language, extra_args, ...plus] = argv; - let codeql = - process.env["SEMMLE_CODE"] ? - path.join(process.env["SEMMLE_CODE"], "target", "intree", `codeql-${language}`, "codeql") - : - "codeql" - ; + let codeql = process.env["SEMMLE_CODE"] + ? path.join( + process.env["SEMMLE_CODE"], + "target", + "intree", + `codeql-${language}`, + "codeql", + ) + : "codeql"; const ram_per_thread = process.platform === "linux" ? 3000 : 2048; const cpus = os.cpus().length; let args: Args = { tests: [], - flags: [ - `--ram=${ram_per_thread * cpus}`, - `-j${cpus}`, - ], + flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`], env: [], build: true, - testing_level: 0 + testing_level: 0, }; parseArgs(args, extra_args); for (let i = 0; i < Math.min(plus.length, args.testing_level); i++) { @@ -72,11 +78,15 @@ function codeqlTestRun(argv: string[]): number { if (args.build && process.env["SEMMLE_CODE"]) { // If SEMMLE_CODE is set, we are in the semmle-code repo, so we build the codeql binary. // Otherwise, we use codeql from PATH. - if (invoke(["python3", "build", `target/intree/codeql-${language}`], {cwd: process.env["SEMMLE_CODE"]}) !== 0) { + if ( + invoke(["python3", "build", `target/intree/codeql-${language}`], { + cwd: process.env["SEMMLE_CODE"], + }) !== 0 + ) { return 1; } } - process.env["CODEQL_CONFIG_FILE"] ||= "." // disable the default implicit config file, but keep an explicit one + process.env["CODEQL_CONFIG_FILE"] ||= "."; // disable the default implicit config file, but keep an explicit one // Set and unset environment variables args.env.forEach((envVar) => { const [key, value] = envVar.split("=", 2); @@ -91,7 +101,9 @@ function codeqlTestRun(argv: string[]): number { process.exit(1); } }); - return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {log_prefix: args.env.join(" ")}); + return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], { + log_prefix: args.env.join(" "), + }); } process.exit(codeqlTestRun(process.argv.slice(2))); diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index 65e834a98425..2f946d41c03d 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -3,63 +3,75 @@ import * as path from "path"; import * as fs from "fs"; function commonDir(paths: string[]): string { - if (paths.length === 0) return ""; - const splitPaths = paths.map(p => p.split(path.sep)); - let i; - for (i = 0; i < splitPaths[0].length; i++) { - if (!splitPaths.every(parts => parts[i] === splitPaths[0][i])) { - break; - } + if (paths.length === 0) return ""; + const splitPaths = paths.map((p) => p.split(path.sep)); + let i; + for (i = 0; i < splitPaths[0].length; i++) { + if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) { + break; + } } const commonParts = splitPaths[0].slice(0, i); - let ret = commonParts.join(path.sep); - if (!fs.existsSync(ret)) { - throw new Error(`Common directory does not exist: ${ret}`); - } - if (!fs.lstatSync(ret).isDirectory()) { - ret = path.dirname(ret); - } - return ret; + let ret = commonParts.join(path.sep); + if (!fs.existsSync(ret)) { + throw new Error(`Common directory does not exist: ${ret}`); + } + if (!fs.lstatSync(ret).isDirectory()) { + ret = path.dirname(ret); + } + return ret; } function forwardCommand(args: string[]): number { - // Avoid infinite recursion - if (args.length == 0) { - console.error("No command provided"); - return 1; - } - const cmd = args[0]; - const envVariable = `__JUST_FORWARD_${cmd}`; - if (process.env[envVariable]) { - console.error(`No common ${cmd} handler found`); - return 1; - } - process.env[envVariable] = "true"; - const cmdArgs = args.slice(1); - // non-positional arguments are flags, repeated + (used by language tests) or environment variable settings - const is_non_positional = /^(-.*|\++|[A-Z_][A-Z_0-9]*=.*)$/; - const flags = cmdArgs.filter(arg => is_non_positional.test(arg)); - const positionalArgs = cmdArgs.filter(arg => !is_non_positional.test(arg)); + // Avoid infinite recursion + if (args.length == 0) { + console.error("No command provided"); + return 1; + } + const cmd = args[0]; + const envVariable = `__JUST_FORWARD_${cmd}`; + if (process.env[envVariable]) { + console.error(`No common ${cmd} handler found`); + return 1; + } + process.env[envVariable] = "true"; + const cmdArgs = args.slice(1); + // non-positional arguments are flags, repeated + (used by language tests) or environment variable settings + const is_non_positional = /^(-.*|\++|[A-Z_][A-Z_0-9]*=.*)$/; + const flags = cmdArgs.filter((arg) => is_non_positional.test(arg)); + const positionalArgs = cmdArgs.filter( + (arg) => !is_non_positional.test(arg), + ); - if (positionalArgs.length === 0) { - console.error("No positional arguments provided"); - return 1; - } + if (positionalArgs.length === 0) { + console.error("No positional arguments provided"); + return 1; + } - const commonPath = commonDir(positionalArgs); - let relativeArgs = positionalArgs.map(arg => path.relative(commonPath, arg) || "."); - if (relativeArgs.length === 1 && relativeArgs[0] === ".") { - relativeArgs = []; - } + const commonPath = commonDir(positionalArgs); + let relativeArgs = positionalArgs.map( + (arg) => path.relative(commonPath, arg) || ".", + ); + if (relativeArgs.length === 1 && relativeArgs[0] === ".") { + relativeArgs = []; + } - const invocation = [process.env["JUST_EXECUTABLE"] || "just", cmd, ...flags, ...relativeArgs]; - console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`); - try { - child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: commonPath }); - } catch (error) { - return 1; - } - return 0; + const invocation = [ + process.env["JUST_EXECUTABLE"] || "just", + cmd, + ...flags, + ...relativeArgs, + ]; + console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd: commonPath, + }); + } catch (error) { + return 1; + } + return 0; } process.exit(forwardCommand(process.argv.slice(2))); From 6e14111337ca02bdc6a65e228ddd6aa4f912f4e9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 7 Jul 2025 17:34:46 +0200 Subject: [PATCH 08/23] Just: use `--all-checks` and `--codeql` special flags, and relativize flags in forwarder --- java/justfile | 3 ++ java/ql/justfile | 6 +++ java/ql/test/justfile | 15 ++++++ misc/just/codeql-test-run.ts | 97 +++++++++++++++++++++++------------- misc/just/forward-command.ts | 21 ++++++-- misc/just/lib.just | 2 +- rust/ql/justfile | 7 ++- swift/ql/justfile | 7 ++- 8 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 java/justfile create mode 100644 java/ql/justfile create mode 100644 java/ql/test/justfile diff --git a/java/justfile b/java/justfile new file mode 100644 index 000000000000..145510c23ea1 --- /dev/null +++ b/java/justfile @@ -0,0 +1,3 @@ +import '../misc/just/lib.just' + +build: (_build "java") diff --git a/java/ql/justfile b/java/ql/justfile new file mode 100644 index 000000000000..ae3d07aba34a --- /dev/null +++ b/java/ql/justfile @@ -0,0 +1,6 @@ +import "../../misc/just/lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +consistency_queries := source_dir() / "consistency-queries" diff --git a/java/ql/test/justfile b/java/ql/test/justfile new file mode 100644 index 000000000000..3ce20e4a32ee --- /dev/null +++ b/java/ql/test/justfile @@ -0,0 +1,15 @@ +import "../justfile" + +base_flags := """\ + CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \ +""" + +all_checks := default_db_checks + """\ + --check-undefined-labels \ + --check-repeated-labels \ + --check-redefined-labels \ + --check-use-before-definition \ + --consistency-queries=""" + consistency_queries + +[no-cd] +test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS) \ No newline at end of file diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts index 746066ecddda..3002e8d394dc 100644 --- a/misc/just/codeql-test-run.ts +++ b/misc/just/codeql-test-run.ts @@ -1,6 +1,7 @@ import * as child_process from "child_process"; import * as path from "path"; import * as os from "os"; +import * as fs from "fs"; function invoke( invocation: string[], @@ -28,65 +29,68 @@ type Args = { tests: string[]; flags: string[]; env: string[]; - build: boolean; - testing_level: number; + codeql: string; + all: boolean; }; function parseArgs(args: Args, argv: string) { - argv.split(/(? { - if (arg === "--no-build") { - args.build = false; - } else if (arg.startsWith("-")) { - args.flags.push(arg); - } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { - args.env.push(arg); - } else if (/^\++$/.test(arg)) { - args.testing_level = Math.max(args.testing_level, arg.length); - } else if (arg !== "") { - args.tests.push(arg); - } - }); + argv.split(/(? arg.replace("\\ ", " ")) + .forEach((arg) => { + if (arg.startsWith("--codeql=")) { + args.codeql = arg.split("=")[1]; + } else if (arg === "+" || arg === "--all-checks") { + args.all = true; + } else if (arg.startsWith("-")) { + args.flags.push(arg); + } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { + args.env.push(arg); + } else if (arg !== "") { + args.tests.push(arg); + } + }); } function codeqlTestRun(argv: string[]): number { - const [language, extra_args, ...plus] = argv; - let codeql = process.env["SEMMLE_CODE"] - ? path.join( - process.env["SEMMLE_CODE"], - "target", - "intree", - `codeql-${language}`, - "codeql", - ) - : "codeql"; + const semmle_code = process.env["SEMMLE_CODE"]; + const [language, base_args, all_args, extra_args] = argv; const ram_per_thread = process.platform === "linux" ? 3000 : 2048; const cpus = os.cpus().length; let args: Args = { tests: [], flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`], env: [], - build: true, - testing_level: 0, + codeql: semmle_code ? "build" : "host", + all: false, }; + parseArgs(args, base_args); parseArgs(args, extra_args); - for (let i = 0; i < Math.min(plus.length, args.testing_level); i++) { - parseArgs(args, plus[i]); + if (args.all) { + parseArgs(args, all_args); + } + if (!semmle_code && (args.codeql === "build" || args.codeql === "built")) { + console.error( + "Using `--codeql=build` or `--codeql=built` requires working with the internal repository", + ); + return 1; } if (args.tests.length === 0) { args.tests.push("."); } - if (args.build && process.env["SEMMLE_CODE"]) { - // If SEMMLE_CODE is set, we are in the semmle-code repo, so we build the codeql binary. - // Otherwise, we use codeql from PATH. + if (args.codeql === "build") { if ( invoke(["python3", "build", `target/intree/codeql-${language}`], { - cwd: process.env["SEMMLE_CODE"], + cwd: semmle_code, }) !== 0 ) { return 1; } } - process.env["CODEQL_CONFIG_FILE"] ||= "."; // disable the default implicit config file, but keep an explicit one + if (args.codeql !== "host") { + // disable the default implicit config file, but keep an explicit one + // this is the same behavior wrt to `--codeql` as the integration test runner + process.env["CODEQL_CONFIG_FILE"] ||= "."; + } // Set and unset environment variables args.env.forEach((envVar) => { const [key, value] = envVar.split("=", 2); @@ -101,6 +105,31 @@ function codeqlTestRun(argv: string[]): number { process.exit(1); } }); + let codeql; + if (args.codeql === "built" || args.codeql === "build") { + codeql = path.join( + semmle_code!, + "target", + "intree", + `codeql-${language}`, + "codeql", + ); + } else if (args.codeql === "host") { + codeql = "codeql"; + } else { + codeql = args.codeql; + if (fs.lstatSync(codeql).isDirectory()) { + codeql = path.join(codeql, "codeql"); + if (process.platform === "win32") { + codeql += ".exe"; + } + } + if (!fs.existsSync(codeql)) { + console.error(`CodeQL executable not found: ${codeql}`); + return 1; + } + } + return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], { log_prefix: args.env.join(" "), }); diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index 2f946d41c03d..7f8264e6e757 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -36,8 +36,8 @@ function forwardCommand(args: string[]): number { } process.env[envVariable] = "true"; const cmdArgs = args.slice(1); - // non-positional arguments are flags, repeated + (used by language tests) or environment variable settings - const is_non_positional = /^(-.*|\++|[A-Z_][A-Z_0-9]*=.*)$/; + // non-positional arguments are flags, + (used by language tests) or environment variable settings + const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/; const flags = cmdArgs.filter((arg) => is_non_positional.test(arg)); const positionalArgs = cmdArgs.filter( (arg) => !is_non_positional.test(arg), @@ -55,11 +55,26 @@ function forwardCommand(args: string[]): number { if (relativeArgs.length === 1 && relativeArgs[0] === ".") { relativeArgs = []; } + let relativeFlags = flags.map((arg) => { + // this might break in specific corner cases, but is good enough for most uses + // workaround if this doesn't work is to not use the forwarder (call just directly in the relevant directory) + if (arg.includes("=") && arg.includes(path.sep)) { + let [flags, flag_arg] = arg.split("=", 2); + flag_arg = flag_arg + .split(path.delimiter) + .map((p) => + path.isAbsolute(p) ? p : path.relative(commonPath, p), + ) + .join(path.delimiter); + return `${flags}=${flag_arg}`; + } + return arg; + }); const invocation = [ process.env["JUST_EXECUTABLE"] || "just", cmd, - ...flags, + ...relativeFlags, ...relativeArgs, ]; console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`); diff --git a/misc/just/lib.just b/misc/just/lib.just index eb16bd5a596d..ffb8731a9c09 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -38,7 +38,7 @@ default_db_checks := """\ """ [no-cd, positional-arguments, no-exit-message] -@_codeql_test LANGUAGE +ARGS: +@_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS: {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" diff --git a/rust/ql/justfile b/rust/ql/justfile index a01e673cdefd..1c903a0110b7 100644 --- a/rust/ql/justfile +++ b/rust/ql/justfile @@ -3,9 +3,8 @@ import "../../misc/just/lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) -db_checks := default_db_checks - -consistency := "--consistency-queries=" + source_dir() / "consistency-queries" +all_checks := default_db_checks + """\ + --consistency-queries=""" + source_dir() / "consistency-queries" [no-cd] -test *ARGS=".": (_codeql_test "rust" ARGS db_checks consistency) \ No newline at end of file +test *ARGS=".": (_codeql_test "rust" "" all_checks ARGS) diff --git a/swift/ql/justfile b/swift/ql/justfile index 37d2d4601ca6..2a60c3810339 100644 --- a/swift/ql/justfile +++ b/swift/ql/justfile @@ -3,13 +3,12 @@ import "../../misc/just/lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) -db_checks := default_db_checks + """\ +all_checks := default_db_checks + """\ --check-repeated-labels \ --check-redefined-labels \ --check-use-before-definition \ -""" + --consistency-queries=""" + source_dir() / "consistency-queries" -consistency := "--consistency-queries=" + source_dir() / "consistency-queries" [no-cd] -test *ARGS=".": (_codeql_test "swift" ARGS db_checks consistency) +test *ARGS=".": (_codeql_test "swift" "" all_checks ARGS) From cb652f3dc8db8a7e6de0c2f367e1bf189364b2ad Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 7 Jul 2025 17:37:19 +0200 Subject: [PATCH 09/23] Just: add `format` to just directory --- misc/just/justfile | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 misc/just/justfile diff --git a/misc/just/justfile b/misc/just/justfile new file mode 100644 index 000000000000..bfa7bed4db2e --- /dev/null +++ b/misc/just/justfile @@ -0,0 +1,2 @@ +format *ARGS=".": + npx prettier --write {{ ARGS }} From fba96c4eae46672c6f2fc6dc05bf9cda55a5fbd5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 7 Jul 2025 17:39:59 +0200 Subject: [PATCH 10/23] Just: add root `lib.just` --- java/justfile | 2 +- java/ql/justfile | 2 +- lib.just | 1 + rust/justfile | 2 +- rust/ql/justfile | 2 +- swift/justfile | 3 +-- swift/ql/integration-tests/justfile | 2 +- swift/ql/justfile | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 lib.just diff --git a/java/justfile b/java/justfile index 145510c23ea1..e67ad738bfb7 100644 --- a/java/justfile +++ b/java/justfile @@ -1,3 +1,3 @@ -import '../misc/just/lib.just' +import '../lib.just' build: (_build "java") diff --git a/java/ql/justfile b/java/ql/justfile index ae3d07aba34a..7a46396fb856 100644 --- a/java/ql/justfile +++ b/java/ql/justfile @@ -1,4 +1,4 @@ -import "../../misc/just/lib.just" +import "../../lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) diff --git a/lib.just b/lib.just new file mode 100644 index 000000000000..0ddd926bcda5 --- /dev/null +++ b/lib.just @@ -0,0 +1 @@ +import "misc/just/lib.just" diff --git a/rust/justfile b/rust/justfile index bcd753087da7..b0f56a7fb00a 100644 --- a/rust/justfile +++ b/rust/justfile @@ -1,4 +1,4 @@ -import '../misc/just/lib.just' +import '../lib.just' install: (_bazel "run" "@codeql//rust:install") diff --git a/rust/ql/justfile b/rust/ql/justfile index 1c903a0110b7..3a8b699e2322 100644 --- a/rust/ql/justfile +++ b/rust/ql/justfile @@ -1,4 +1,4 @@ -import "../../misc/just/lib.just" +import "../../lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) diff --git a/swift/justfile b/swift/justfile index d63d7f32e4fd..cd7cf3ce583c 100644 --- a/swift/justfile +++ b/swift/justfile @@ -1,4 +1,4 @@ -import '../misc/just/lib.just' +import '../lib.just' install: (_bazel "run" "@codeql//swift:install") @@ -12,4 +12,3 @@ generate: (_bazel "run" "@codeql//swift/codegen") fi format ARGS=".": _check_clang_format (_run "clang-format" "-i" ("$(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')")) - diff --git a/swift/ql/integration-tests/justfile b/swift/ql/integration-tests/justfile index c8fad074a3f9..370f7ef87794 100644 --- a/swift/ql/integration-tests/justfile +++ b/swift/ql/integration-tests/justfile @@ -1,4 +1,4 @@ -import "../../../misc/just/lib.just" +import "../../../lib.just" [no-cd] diff --git a/swift/ql/justfile b/swift/ql/justfile index 2a60c3810339..7fe1fd1faa5e 100644 --- a/swift/ql/justfile +++ b/swift/ql/justfile @@ -1,4 +1,4 @@ -import "../../misc/just/lib.just" +import "../../lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) From c4305151c3274f63e17c7e8ac8177fcd0adfb0ce Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 10:52:15 +0200 Subject: [PATCH 11/23] Just: simplify forwarder using `--justfile` --- misc/just/forward-command.ts | 53 +++++++++++------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index 7f8264e6e757..c64f974c0cb5 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -2,24 +2,23 @@ import * as child_process from "child_process"; import * as path from "path"; import * as fs from "fs"; -function commonDir(paths: string[]): string { +function commonJustfile(paths: string[]): string { if (paths.length === 0) return ""; const splitPaths = paths.map((p) => p.split(path.sep)); - let i; - for (i = 0; i < splitPaths[0].length; i++) { + let justfile: string | undefined = undefined; + for (let i = 0; i < splitPaths[0].length; i++) { + let candidate = path.join(...splitPaths[0].slice(0, i), "justfile"); + if (fs.existsSync(candidate)) { + justfile = candidate; + } if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) { break; } } - const commonParts = splitPaths[0].slice(0, i); - let ret = commonParts.join(path.sep); - if (!fs.existsSync(ret)) { - throw new Error(`Common directory does not exist: ${ret}`); - } - if (!fs.lstatSync(ret).isDirectory()) { - ret = path.dirname(ret); + if (justfile === undefined) { + throw new Error("No common justfile found"); } - return ret; + return justfile; } function forwardCommand(args: string[]): number { @@ -48,40 +47,20 @@ function forwardCommand(args: string[]): number { return 1; } - const commonPath = commonDir(positionalArgs); - let relativeArgs = positionalArgs.map( - (arg) => path.relative(commonPath, arg) || ".", - ); - if (relativeArgs.length === 1 && relativeArgs[0] === ".") { - relativeArgs = []; - } - let relativeFlags = flags.map((arg) => { - // this might break in specific corner cases, but is good enough for most uses - // workaround if this doesn't work is to not use the forwarder (call just directly in the relevant directory) - if (arg.includes("=") && arg.includes(path.sep)) { - let [flags, flag_arg] = arg.split("=", 2); - flag_arg = flag_arg - .split(path.delimiter) - .map((p) => - path.isAbsolute(p) ? p : path.relative(commonPath, p), - ) - .join(path.delimiter); - return `${flags}=${flag_arg}`; - } - return arg; - }); + const justfile = commonJustfile(positionalArgs); const invocation = [ process.env["JUST_EXECUTABLE"] || "just", + "--justfile", + justfile, cmd, - ...relativeFlags, - ...relativeArgs, + ...flags, + ...positionalArgs, ]; - console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`); + console.log(`-> ${justfile}: ${invocation.slice(1).join(" ")}`); try { child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", - cwd: commonPath, }); } catch (error) { return 1; From acc7e3f32d40fbea079a11557ef6d596de6e0246 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 10:52:37 +0200 Subject: [PATCH 12/23] Just: add `generate` prerequisite to rust ql tests --- misc/just/lib.just | 1 + rust/ql/justfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/just/lib.just b/misc/just/lib.just index ffb8731a9c09..437418d8841b 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -39,6 +39,7 @@ default_db_checks := """\ [no-cd, positional-arguments, no-exit-message] @_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS: + #$language-tests|{{ LANGUAGE }}|{{{{ BASE_FLAGS }}|{{ ALL_CHECKS_FLAGS }} {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" diff --git a/rust/ql/justfile b/rust/ql/justfile index 3a8b699e2322..19f867b764cc 100644 --- a/rust/ql/justfile +++ b/rust/ql/justfile @@ -7,4 +7,4 @@ all_checks := default_db_checks + """\ --consistency-queries=""" + source_dir() / "consistency-queries" [no-cd] -test *ARGS=".": (_codeql_test "rust" "" all_checks ARGS) +test *ARGS=".": (_just "generate") (_codeql_test "rust" "" all_checks ARGS) From e8bcbbd6df11839fb4d2e59720d4bd4792ae4320 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 14:06:14 +0200 Subject: [PATCH 13/23] Just: add `language-tests.ts` helper --- java/ql/test/justfile | 3 ++- misc/just/codeql-test-run.ts | 6 +++++- misc/just/forward-command.ts | 4 ++-- misc/just/language-tests.ts | 31 +++++++++++++++++++++++++++++++ misc/just/lib.just | 4 +++- rust/codegen/codegen.sh | 2 +- rust/justfile | 6 +++++- rust/ql/justfile | 6 +----- rust/ql/test/justfile | 7 +++++++ swift/justfile | 8 ++++++++ swift/ql/justfile | 10 +--------- swift/ql/test/justfile | 11 +++++++++++ 12 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 misc/just/language-tests.ts create mode 100644 rust/ql/test/justfile create mode 100644 swift/ql/test/justfile diff --git a/java/ql/test/justfile b/java/ql/test/justfile index 3ce20e4a32ee..f31d7c3c2139 100644 --- a/java/ql/test/justfile +++ b/java/ql/test/justfile @@ -12,4 +12,5 @@ all_checks := default_db_checks + """\ --consistency-queries=""" + consistency_queries [no-cd] -test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS) \ No newline at end of file +test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS) + diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts index 3002e8d394dc..b1c39aa4bbc6 100644 --- a/misc/just/codeql-test-run.ts +++ b/misc/just/codeql-test-run.ts @@ -79,7 +79,7 @@ function codeqlTestRun(argv: string[]): number { } if (args.codeql === "build") { if ( - invoke(["python3", "build", `target/intree/codeql-${language}`], { + invoke([process.env["JUST_EXECUTABLE"] || "just", language, "build"], { cwd: semmle_code, }) !== 0 ) { @@ -114,6 +114,10 @@ function codeqlTestRun(argv: string[]): number { `codeql-${language}`, "codeql", ); + if (!fs.existsSync(codeql)) { + console.error(`CodeQL executable not found: ${codeql}`); + return 1; + } } else if (args.codeql === "host") { codeql = "codeql"; } else { diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index c64f974c0cb5..2d6d1e8ed80e 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -7,7 +7,7 @@ function commonJustfile(paths: string[]): string { const splitPaths = paths.map((p) => p.split(path.sep)); let justfile: string | undefined = undefined; for (let i = 0; i < splitPaths[0].length; i++) { - let candidate = path.join(...splitPaths[0].slice(0, i), "justfile"); + let candidate = path.resolve(path.join(splitPaths[0].slice(0, i).join(path.sep), "justfile")); if (fs.existsSync(candidate)) { justfile = candidate; } @@ -57,7 +57,7 @@ function forwardCommand(args: string[]): number { ...flags, ...positionalArgs, ]; - console.log(`-> ${justfile}: ${invocation.slice(1).join(" ")}`); + console.log(`-> just ${invocation.slice(1).join(" ")}`); try { child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", diff --git a/misc/just/language-tests.ts b/misc/just/language-tests.ts new file mode 100644 index 000000000000..c3626109e03d --- /dev/null +++ b/misc/just/language-tests.ts @@ -0,0 +1,31 @@ +import * as path from "path"; +import * as process from "process" +import * as child_process from "child_process"; + +function languageTests(argv: string[]): number { + const [extra_args, dir, ...relativeRoots] = argv; + const semmle_code = process.env["SEMMLE_CODE"]!; + let roots = relativeRoots.map((root) => path.relative(semmle_code, path.join(dir, root))); + const invocation = [ + process.env["JUST_EXECUTABLE"] || "just", + "--justfile", + path.join(roots[0], "justfile"), + "test", + "--all-checks", + "--codeql=built", + ...extra_args.split(" "), + ...roots, + ]; + console.log(`-> just ${invocation.slice(1).join(" ")}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd: semmle_code, + }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(languageTests(process.argv.slice(2))); diff --git a/misc/just/lib.just b/misc/just/lib.just index 437418d8841b..5101ae48f2c5 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -39,9 +39,11 @@ default_db_checks := """\ [no-cd, positional-arguments, no-exit-message] @_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS: - #$language-tests|{{ LANGUAGE }}|{{{{ BASE_FLAGS }}|{{ ALL_CHECKS_FLAGS }} {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" +[no-cd, positional-arguments, no-exit-message] +@_language_tests EXTRA_ARGS SOURCE_DIR +ROOTS: _require_semmle_code + {{ tsx }} "{{ source_dir() }}/language-tests.ts" "$@" [no-cd, no-exit-message] _ql_format +ARGS: (_maybe_build "nolang") diff --git a/rust/codegen/codegen.sh b/rust/codegen/codegen.sh index 2d415009aed8..726ff138db78 100755 --- a/rust/codegen/codegen.sh +++ b/rust/codegen/codegen.sh @@ -2,7 +2,7 @@ set -eu -source misc/bazel/runfiles.sh 2>/dev/null || source external/ql+/misc/bazel/runfiles.sh +source misc/bazel/runfiles.sh 2>/dev/null || source ../ql+/misc/bazel/runfiles.sh ast_generator="$(rlocation "$1")" grammar_file="$(rlocation "$2")" diff --git a/rust/justfile b/rust/justfile index b0f56a7fb00a..8973c00bf941 100644 --- a/rust/justfile +++ b/rust/justfile @@ -1,7 +1,8 @@ -import '../lib.just' +import '../justfile' install: (_bazel "run" "@codeql//rust:install") +[group('build')] build: generate (_build "rust") generate: (_bazel "run" "@codeql//rust/codegen") @@ -9,3 +10,6 @@ generate: (_bazel "run" "@codeql//rust/codegen") lint: (_run "python3" "lint.py") format: (_run "python3" "lint.py" "--format-only") + +[group('test')] +language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test') diff --git a/rust/ql/justfile b/rust/ql/justfile index 19f867b764cc..7a46396fb856 100644 --- a/rust/ql/justfile +++ b/rust/ql/justfile @@ -3,8 +3,4 @@ import "../../lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) -all_checks := default_db_checks + """\ - --consistency-queries=""" + source_dir() / "consistency-queries" - -[no-cd] -test *ARGS=".": (_just "generate") (_codeql_test "rust" "" all_checks ARGS) +consistency_queries := source_dir() / "consistency-queries" diff --git a/rust/ql/test/justfile b/rust/ql/test/justfile new file mode 100644 index 000000000000..0b1b075fbbb0 --- /dev/null +++ b/rust/ql/test/justfile @@ -0,0 +1,7 @@ +import "../justfile" + +all_checks := default_db_checks + """\ + --consistency-queries=""" + consistency_queries + +[no-cd] +test *ARGS=".": (_codeql_test "rust" "" all_checks ARGS) diff --git a/swift/justfile b/swift/justfile index cd7cf3ce583c..2e140fee39c6 100644 --- a/swift/justfile +++ b/swift/justfile @@ -2,6 +2,7 @@ import '../lib.just' install: (_bazel "run" "@codeql//swift:install") +[group('build')] build: (_build "swift") generate: (_bazel "run" "@codeql//swift/codegen") @@ -12,3 +13,10 @@ generate: (_bazel "run" "@codeql//swift/codegen") fi format ARGS=".": _check_clang_format (_run "clang-format" "-i" ("$(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')")) +import "../../ql/swift/ql/justfile" + +[group('test')] +language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test') + +[group('test')] +extra-tests: (_sembuild "target/test/check-queries-swift") (_sembuild "target/test/check-db-upgrades-swift") (_sembuild "target/test/check-db-downgrades-swift") diff --git a/swift/ql/justfile b/swift/ql/justfile index 7fe1fd1faa5e..7a46396fb856 100644 --- a/swift/ql/justfile +++ b/swift/ql/justfile @@ -3,12 +3,4 @@ import "../../lib.just" [no-cd] format *ARGS=".": (_ql_format ARGS) -all_checks := default_db_checks + """\ - --check-repeated-labels \ - --check-redefined-labels \ - --check-use-before-definition \ - --consistency-queries=""" + source_dir() / "consistency-queries" - - -[no-cd] -test *ARGS=".": (_codeql_test "swift" "" all_checks ARGS) +consistency_queries := source_dir() / "consistency-queries" diff --git a/swift/ql/test/justfile b/swift/ql/test/justfile new file mode 100644 index 000000000000..b27ef6f52ed7 --- /dev/null +++ b/swift/ql/test/justfile @@ -0,0 +1,11 @@ +import "../justfile" + +all_checks := default_db_checks + """\ + --check-repeated-labels \ + --check-redefined-labels \ + --check-use-before-definition \ + --consistency-queries=""" + consistency_queries + + +[no-cd] +test *ARGS=".": (_codeql_test "swift" "" all_checks ARGS) From bb467d4abfb61f7677dd4134dd33cad00228967d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 15:40:35 +0200 Subject: [PATCH 14/23] Just: fix CI build for rust --- rust/justfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rust/justfile b/rust/justfile index 8973c00bf941..4af392ff2fb3 100644 --- a/rust/justfile +++ b/rust/justfile @@ -1,9 +1,8 @@ -import '../justfile' +import '../lib.just' install: (_bazel "run" "@codeql//rust:install") -[group('build')] -build: generate (_build "rust") +build: (_build "rust") generate: (_bazel "run" "@codeql//rust/codegen") From 8ba7efd4556a7b33fd8631b013cbb174fe5fc8ae Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 15:41:18 +0200 Subject: [PATCH 15/23] Just: fix mono-argument case for argumentless recipes --- misc/just/forward-command.ts | 41 ++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index 2d6d1e8ed80e..ed01bf3c4c56 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -6,8 +6,8 @@ function commonJustfile(paths: string[]): string { if (paths.length === 0) return ""; const splitPaths = paths.map((p) => p.split(path.sep)); let justfile: string | undefined = undefined; - for (let i = 0; i < splitPaths[0].length; i++) { - let candidate = path.resolve(path.join(splitPaths[0].slice(0, i).join(path.sep), "justfile")); + for (let i = 0; i <= splitPaths[0].length; i++) { + let candidate = path.join(splitPaths[0].slice(0, i).join(path.sep), "justfile"); if (fs.existsSync(candidate)) { justfile = candidate; } @@ -42,25 +42,34 @@ function forwardCommand(args: string[]): number { (arg) => !is_non_positional.test(arg), ); - if (positionalArgs.length === 0) { - console.error("No positional arguments provided"); - return 1; + const justfile = commonJustfile(positionalArgs.length !== 0 ? positionalArgs : ["."]); + let cwd: string | undefined; + let invocation = [process.env["JUST_EXECUTABLE"] || "just"]; + if (positionalArgs.length === 1 && justfile == path.join(positionalArgs[0], "justfile")) { + // If there's only one positional argument and it matches the justfile path, suppress arguments + // so for example `just build ql/rust` becomes `just build` in the `ql/rust` directory + cwd = positionalArgs[0]; + invocation.push( + cmd, + ...flags, + ); + console.log(`-> cd ${cwd}; just ${invocation.slice(1).join(" ")}`); + } else { + cwd = undefined; + invocation.push( + "--justfile", + justfile, + cmd, + ...flags, + ...positionalArgs, + ); + console.log(`-> just ${invocation.slice(1).join(" ")}`); } - const justfile = commonJustfile(positionalArgs); - - const invocation = [ - process.env["JUST_EXECUTABLE"] || "just", - "--justfile", - justfile, - cmd, - ...flags, - ...positionalArgs, - ]; - console.log(`-> just ${invocation.slice(1).join(" ")}`); try { child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", + cwd, }); } catch (error) { return 1; From aa09288462cb846a754377942a19af4e04e6a255 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 15:49:23 +0200 Subject: [PATCH 16/23] Just: introduce aliases --- misc/just/forward.just | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/misc/just/forward.just b/misc/just/forward.just index 64af38d272fa..5d3664da221d 100644 --- a/misc/just/forward.just +++ b/misc/just/forward.just @@ -5,24 +5,30 @@ import "lib.just" _forward := tsx + ' "' + source_dir() + '/forward-command.ts"' +alias t := test +alias b := build +alias g := generate +alias gen := generate +alias f := format +alias l := lint [no-cd, positional-arguments, no-exit-message] -@test *ARGS=".": +@test *ARGS: {{ _forward }} test "$@" [no-cd, positional-arguments, no-exit-message] -@build *ARGS=".": +@build *ARGS: {{ _forward }} build "$@" [no-cd, positional-arguments, no-exit-message] -@generate *ARGS=".": +@generate *ARGS: {{ _forward }} generate "$@" [no-cd, positional-arguments, no-exit-message] -@lint *ARGS=".": +@lint *ARGS: {{ _forward }} lint "$@" [no-cd, positional-arguments, no-exit-message] -@format *ARGS=".": +@format *ARGS: {{ _forward }} format "$@" From 7f72f872041406eff5cc48006e1131eca228e725 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 15:50:37 +0200 Subject: [PATCH 17/23] Just: fix `just rust format` and similar --- misc/just/lib.just | 8 +++++--- rust/justfile | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/misc/just/lib.just b/misc/just/lib.just index 5101ae48f2c5..b16ebdab7f8c 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -52,7 +52,7 @@ _ql_format +ARGS: (_maybe_build "nolang") [no-cd, no-exit-message] _bazel COMMAND *ARGS: - {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/tools/bazel"' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { 'cd "$SEMMLE_CODE"; tools/bazel' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} [no-cd, no-exit-message] _sembuild *ARGS: (_run_in_semmle_code "./build" ARGS) @@ -65,9 +65,11 @@ _run +ARGS: {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }} [no-cd] -_run_in_semmle_code +ARGS: _require_semmle_code - {{ cmd_sep }}cd "$SEMMLE_CODE"; {{ ARGS }}{{ cmd_sep }} +_run_in DIR +ARGS: + {{ cmd_sep }}cd "{{ DIR }}"; {{ ARGS }}{{ cmd_sep }} +[no-cd] +_run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS) [no-cd, positional-arguments, no-exit-message] _just +ARGS: diff --git a/rust/justfile b/rust/justfile index 4af392ff2fb3..85b008abbf5b 100644 --- a/rust/justfile +++ b/rust/justfile @@ -6,9 +6,9 @@ build: (_build "rust") generate: (_bazel "run" "@codeql//rust/codegen") -lint: (_run "python3" "lint.py") +lint: (_run_in source_dir() "python3" "lint.py") -format: (_run "python3" "lint.py" "--format-only") +format: (_run_in source_dir() "python3" "lint.py" "--format-only") [group('test')] language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test') From b8b01ce71c6d3e2ace79abfedd3fa0158a9f4fea Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 15:53:51 +0200 Subject: [PATCH 18/23] Just: rename `_build` to `_build_dist` --- java/justfile | 2 +- misc/just/lib.just | 6 +++--- rust/justfile | 2 +- swift/justfile | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/justfile b/java/justfile index e67ad738bfb7..a6618b527848 100644 --- a/java/justfile +++ b/java/justfile @@ -1,3 +1,3 @@ import '../lib.just' -build: (_build "java") +build: (_build_dist "java") diff --git a/misc/just/lib.just b/misc/just/lib.just index b16ebdab7f8c..35c6712b9ee8 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -24,10 +24,10 @@ import 'semmle-code-stub.just' exit 1 ''' } else { "" } }} -_build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE) +_build_dist LANGUAGE: _require_semmle_code (_maybe_build_dist LANGUAGE) [no-exit-message] -_maybe_build LANGUAGE: +_maybe_build_dist LANGUAGE: {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} @@ -46,7 +46,7 @@ default_db_checks := """\ {{ tsx }} "{{ source_dir() }}/language-tests.ts" "$@" [no-cd, no-exit-message] -_ql_format +ARGS: (_maybe_build "nolang") +_ql_format +ARGS: (_maybe_build_dist "nolang") {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} diff --git a/rust/justfile b/rust/justfile index 85b008abbf5b..90706bd8e9c6 100644 --- a/rust/justfile +++ b/rust/justfile @@ -2,7 +2,7 @@ import '../lib.just' install: (_bazel "run" "@codeql//rust:install") -build: (_build "rust") +build: (_build_dist "rust") generate: (_bazel "run" "@codeql//rust/codegen") diff --git a/swift/justfile b/swift/justfile index 2e140fee39c6..f09fea32e525 100644 --- a/swift/justfile +++ b/swift/justfile @@ -3,7 +3,7 @@ import '../lib.just' install: (_bazel "run" "@codeql//swift:install") [group('build')] -build: (_build "swift") +build: (_build_dist "swift") generate: (_bazel "run" "@codeql//swift/codegen") From bd003c58a8e489db1315bda7a776444bf095fc96 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 8 Jul 2025 17:22:27 +0200 Subject: [PATCH 19/23] Just: add `_if_not_on_ci_just` helper, and add generation prerequisites --- misc/just/lib.just | 10 +++++++++- rust/justfile | 2 +- rust/ql/integration-tests/justfile | 5 +++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/misc/just/lib.just b/misc/just/lib.just index 35c6712b9ee8..94ee77efaa50 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -72,8 +72,16 @@ _run_in DIR +ARGS: _run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS) [no-cd, positional-arguments, no-exit-message] -_just +ARGS: +@_just +ARGS: + echo "-> just $@" "{{ JUST_EXECUTABLE }}" "$@" +[no-cd, positional-arguments] +@_if_not_on_ci_just +ARGS: + if [ "${GITHUB_ACTIONS:-}" != "true" ]; then \ + echo "-> just $@"; \ + "$JUST_EXECUTABLE" "$@"; \ + fi + [no-cd] _black *ARGS=".": (_run "uv" "run" "black" ARGS) diff --git a/rust/justfile b/rust/justfile index 90706bd8e9c6..a09cabd1d168 100644 --- a/rust/justfile +++ b/rust/justfile @@ -2,7 +2,7 @@ import '../lib.just' install: (_bazel "run" "@codeql//rust:install") -build: (_build_dist "rust") +build: (_if_not_on_ci_just "generate" source_dir()) (_build_dist "rust") generate: (_bazel "run" "@codeql//rust/codegen") diff --git a/rust/ql/integration-tests/justfile b/rust/ql/integration-tests/justfile index c8fad074a3f9..7f5aab6c508c 100644 --- a/rust/ql/integration-tests/justfile +++ b/rust/ql/integration-tests/justfile @@ -1,8 +1,9 @@ -import "../../../misc/just/lib.just" + +import "../../../lib.just" [no-cd] -test *ARGS=".": (_just "generate") (_integration_test ARGS) +test *ARGS=".": (_if_not_on_ci_just "generate" source_dir()) (_integration_test ARGS) # TODO in separate PR # [no-cd] From c9cda74195c89d5d27337c7029617f686623661a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 9 Jul 2025 14:47:38 +0200 Subject: [PATCH 20/23] Just: allow mixing different verb implementations This allows to mix different verb implementations in a single invocation, for example: ``` just build ql/rust ql/java just test ql/rust/ql/test/some/test ql/rust/ql/integrartion-test/some other ``` If a common justfile recipe is found, it is used for all arguments in one go. If on the other hand no common justfile recipe is found, each argument is processed separately in sequence. This does require that any flags passed are compatible with all recipes involved (like is the case for `--learn` or `--codeql=built` for language and integration tests). --- misc/just/forward-command.ts | 128 ++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 41 deletions(-) diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index ed01bf3c4c56..b02835a7db6d 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -1,73 +1,119 @@ import * as child_process from "child_process"; import * as path from "path"; -import * as fs from "fs"; -function commonJustfile(paths: string[]): string { +const just = process.env["JUST_EXECUTABLE"]!; + +console.debug = (...args: any[]) => {} // comment out to debug script + +function checkJustCommand(justfile: string, command: string, postitionalArgs: string[]): boolean { + let {cwd, args} = getJustContext(justfile, command, [], postitionalArgs); + console.debug(`Checking: ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(", ")}`); + const res = child_process.spawnSync(just, ["--dry-run", ...args], { + stdio: ["ignore", "ignore", "pipe"], + encoding: "utf8", + cwd, + }); + console.debug("result:", res); + // avoid having the forwarder find itself + return res.status === 0 && !res.stderr.includes(`forward-command.ts" ${command} "$@"`); +} + +function commonPath(paths: string[]): string { if (paths.length === 0) return ""; + if (paths.length === 1) return paths[0]; const splitPaths = paths.map((p) => p.split(path.sep)); - let justfile: string | undefined = undefined; - for (let i = 0; i <= splitPaths[0].length; i++) { - let candidate = path.join(splitPaths[0].slice(0, i).join(path.sep), "justfile"); - if (fs.existsSync(candidate)) { - justfile = candidate; - } + let i; + for (i = 0; i <= splitPaths[0].length; i++) { if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) { break; } } - if (justfile === undefined) { - throw new Error("No common justfile found"); + return splitPaths[0].slice(0, i).join(path.sep); +} + +function findJustfile(command: string, paths: string[]): string | undefined { + const common = commonPath(paths); + for (let p = common;; p = path.dirname(p)) { + const candidate = path.join(p, "justfile"); + if (checkJustCommand(candidate, command, paths)) { + return candidate; + } + if (p === "/" || p === ".") { + return undefined; + } } - return justfile; } function forwardCommand(args: string[]): number { - // Avoid infinite recursion if (args.length == 0) { console.error("No command provided"); return 1; } - const cmd = args[0]; - const envVariable = `__JUST_FORWARD_${cmd}`; - if (process.env[envVariable]) { - console.error(`No common ${cmd} handler found`); - return 1; - } - process.env[envVariable] = "true"; - const cmdArgs = args.slice(1); + return forward(args[0], args.slice(1)); +} + +function forward(cmd: string, args: string[]): number { // non-positional arguments are flags, + (used by language tests) or environment variable settings const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/; - const flags = cmdArgs.filter((arg) => is_non_positional.test(arg)); - const positionalArgs = cmdArgs.filter( + const flags = args.filter((arg) => is_non_positional.test(arg)); + const positionalArgs = args.filter( (arg) => !is_non_positional.test(arg), ); - const justfile = commonJustfile(positionalArgs.length !== 0 ? positionalArgs : ["."]); - let cwd: string | undefined; - let invocation = [process.env["JUST_EXECUTABLE"] || "just"]; + const justfile = findJustfile(cmd, positionalArgs.length !== 0 ? positionalArgs : ["."]); + if (!justfile) { + if (positionalArgs.length <= 1) { + console.error(`No justfile found for ${cmd}${positionalArgs.length === 0 ? "" : " on " + positionalArgs[0]}`); + return 1; + } + console.log(`no common justfile recipe found for ${cmd} for all arguments, trying one argument at a time`); + const runs: [string, string | undefined][] = positionalArgs.map(arg => [arg, findJustfile(cmd, [arg])]); + for (const [arg, justfile] of runs) { + if (!justfile) { + console.error(`No justfile found for ${cmd} on ${arg}`); + return 1; + } + } + for (const [arg, justfile] of runs) { + if (invokeJust(justfile!, cmd, flags, [arg]) !== 0) { + return 1; + } + } + return 0; + } + return invokeJust(justfile, cmd, flags, positionalArgs); +} + +function getJustContext(justfile: string, cmd: string, flags: string[], positionalArgs: string[]): {args: string[], cwd?: string} { if (positionalArgs.length === 1 && justfile == path.join(positionalArgs[0], "justfile")) { // If there's only one positional argument and it matches the justfile path, suppress arguments // so for example `just build ql/rust` becomes `just build` in the `ql/rust` directory - cwd = positionalArgs[0]; - invocation.push( - cmd, - ...flags, - ); - console.log(`-> cd ${cwd}; just ${invocation.slice(1).join(" ")}`); + return { + cwd: positionalArgs[0], + args: [ + cmd, + ...flags, + ], + }; } else { - cwd = undefined; - invocation.push( - "--justfile", - justfile, - cmd, - ...flags, - ...positionalArgs, - ); - console.log(`-> just ${invocation.slice(1).join(" ")}`); + return { + cwd: undefined, + args: [ + "--justfile", + justfile, + cmd, + ...flags, + ...positionalArgs, + ], + }; } +} +function invokeJust(justfile: string, cmd: string, flags: string[], positionalArgs: string[]): number { + const { cwd, args } = getJustContext(justfile, cmd, flags, positionalArgs); + console.log(`-> ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(" ")}`); try { - child_process.execFileSync(invocation[0], invocation.slice(1), { + child_process.execFileSync(just, args, { stdio: "inherit", cwd, }); From 103745b5d25948d74d0fc8d1a7244cbdef08e741 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 10 Jul 2025 10:34:35 +0200 Subject: [PATCH 21/23] Just: group just invocations in forwarder and improve logging --- misc/just/codeql-test-run.ts | 54 +++++++++++++++---------- misc/just/forward-command.ts | 77 ++++++++++++++++-------------------- misc/just/lib.just | 1 + 3 files changed, 70 insertions(+), 62 deletions(-) diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts index b1c39aa4bbc6..b8f892738ed5 100644 --- a/misc/just/codeql-test-run.ts +++ b/misc/just/codeql-test-run.ts @@ -3,6 +3,14 @@ import * as path from "path"; import * as os from "os"; import * as fs from "fs"; +const vars = { + just: process.env["JUST_EXECUTABLE"] || "just", + error: process.env["JUST_ERROR"] || "error", + cmd_begin: process.env["CMD_BEGIN"] || "", + cmd_end: process.env["CMD_END"] || "", + semmle_code: process.env["SEMMLE_CODE"], +} + function invoke( invocation: string[], options: { cwd?: string; log_prefix?: string } = {}, @@ -12,7 +20,7 @@ function invoke( ? `${options.log_prefix} ` : ""; console.log( - `${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`, + `${vars.cmd_begin}${log_prefix}${invocation.join(" ")}${vars.cmd_end}`, ); try { child_process.execFileSync(invocation[0], invocation.slice(1), { @@ -33,6 +41,12 @@ type Args = { all: boolean; }; +const old_console_error = console.error; + +console.error = (message: string) => { + old_console_error(vars.error + message); +}; + function parseArgs(args: Args, argv: string) { argv.split(/(? arg.replace("\\ ", " ")) @@ -52,7 +66,6 @@ function parseArgs(args: Args, argv: string) { } function codeqlTestRun(argv: string[]): number { - const semmle_code = process.env["SEMMLE_CODE"]; const [language, base_args, all_args, extra_args] = argv; const ram_per_thread = process.platform === "linux" ? 3000 : 2048; const cpus = os.cpus().length; @@ -60,7 +73,7 @@ function codeqlTestRun(argv: string[]): number { tests: [], flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`], env: [], - codeql: semmle_code ? "build" : "host", + codeql: vars.semmle_code ? "build" : "host", all: false, }; parseArgs(args, base_args); @@ -68,7 +81,7 @@ function codeqlTestRun(argv: string[]): number { if (args.all) { parseArgs(args, all_args); } - if (!semmle_code && (args.codeql === "build" || args.codeql === "built")) { + if (!vars.semmle_code && (args.codeql === "build" || args.codeql === "built")) { console.error( "Using `--codeql=build` or `--codeql=built` requires working with the internal repository", ); @@ -79,8 +92,8 @@ function codeqlTestRun(argv: string[]): number { } if (args.codeql === "build") { if ( - invoke([process.env["JUST_EXECUTABLE"] || "just", language, "build"], { - cwd: semmle_code, + invoke([vars.just, language, "build"], { + cwd: vars.semmle_code, }) !== 0 ) { return 1; @@ -106,32 +119,33 @@ function codeqlTestRun(argv: string[]): number { } }); let codeql; + function check_codeql() { + if (!fs.existsSync(codeql)) { + console.error(`CodeQL executable not found: ${codeql}`); + process.exit(1); + } + } if (args.codeql === "built" || args.codeql === "build") { codeql = path.join( - semmle_code!, + vars.semmle_code!, "target", "intree", `codeql-${language}`, "codeql", ); - if (!fs.existsSync(codeql)) { - console.error(`CodeQL executable not found: ${codeql}`); - return 1; - } + check_codeql(); } else if (args.codeql === "host") { codeql = "codeql"; } else { codeql = args.codeql; - if (fs.lstatSync(codeql).isDirectory()) { - codeql = path.join(codeql, "codeql"); - if (process.platform === "win32") { - codeql += ".exe"; - } - } - if (!fs.existsSync(codeql)) { - console.error(`CodeQL executable not found: ${codeql}`); - return 1; + check_codeql(); + } + if (fs.lstatSync(codeql).isDirectory()) { + codeql = path.join(codeql, "codeql"); + if (process.platform === "win32") { + codeql += ".exe"; } + check_codeql(); } return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], { diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts index b02835a7db6d..c2f8682d1d59 100644 --- a/misc/just/forward-command.ts +++ b/misc/just/forward-command.ts @@ -1,14 +1,26 @@ import * as child_process from "child_process"; import * as path from "path"; +import * as fs from "fs"; -const just = process.env["JUST_EXECUTABLE"]!; +const vars = { + just: process.env["JUST_EXECUTABLE"] || "just", + error: process.env["JUST_ERROR"] || "", +}; console.debug = (...args: any[]) => {} // comment out to debug script +const old_console_error = console.error; +console.error = (message: string) => { + old_console_error(vars.error + message); +}; + function checkJustCommand(justfile: string, command: string, postitionalArgs: string[]): boolean { + if (!fs.existsSync(justfile)) { + return false; + } let {cwd, args} = getJustContext(justfile, command, [], postitionalArgs); console.debug(`Checking: ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(", ")}`); - const res = child_process.spawnSync(just, ["--dry-run", ...args], { + const res = child_process.spawnSync(vars.just, ["--dry-run", ...args], { stdio: ["ignore", "ignore", "pipe"], encoding: "utf8", cwd, @@ -18,24 +30,10 @@ function checkJustCommand(justfile: string, command: string, postitionalArgs: st return res.status === 0 && !res.stderr.includes(`forward-command.ts" ${command} "$@"`); } -function commonPath(paths: string[]): string { - if (paths.length === 0) return ""; - if (paths.length === 1) return paths[0]; - const splitPaths = paths.map((p) => p.split(path.sep)); - let i; - for (i = 0; i <= splitPaths[0].length; i++) { - if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) { - break; - } - } - return splitPaths[0].slice(0, i).join(path.sep); -} - -function findJustfile(command: string, paths: string[]): string | undefined { - const common = commonPath(paths); - for (let p = common;; p = path.dirname(p)) { +function findJustfile(command: string, arg: string): string | undefined { + for (let p = arg;; p = path.dirname(p)) { const candidate = path.join(p, "justfile"); - if (checkJustCommand(candidate, command, paths)) { + if (checkJustCommand(candidate, command, [arg])) { return candidate; } if (p === "/" || p === ".") { @@ -59,29 +57,26 @@ function forward(cmd: string, args: string[]): number { const positionalArgs = args.filter( (arg) => !is_non_positional.test(arg), ); - - const justfile = findJustfile(cmd, positionalArgs.length !== 0 ? positionalArgs : ["."]); - if (!justfile) { - if (positionalArgs.length <= 1) { - console.error(`No justfile found for ${cmd}${positionalArgs.length === 0 ? "" : " on " + positionalArgs[0]}`); + let justfiles: Map = new Map(); + for(const arg of positionalArgs.length > 0 ? positionalArgs : ["."]) { + const justfile = findJustfile(cmd, arg); + if (!justfile) { + console.error(`No justfile found for ${cmd} on ${arg}`); return 1; } - console.log(`no common justfile recipe found for ${cmd} for all arguments, trying one argument at a time`); - const runs: [string, string | undefined][] = positionalArgs.map(arg => [arg, findJustfile(cmd, [arg])]); - for (const [arg, justfile] of runs) { - if (!justfile) { - console.error(`No justfile found for ${cmd} on ${arg}`); - return 1; - } - } - for (const [arg, justfile] of runs) { - if (invokeJust(justfile!, cmd, flags, [arg]) !== 0) { - return 1; - } + justfiles.set(justfile, [...justfiles.get(justfile) || [], arg]); + } + const invocations = Array.from(justfiles.entries()).map(([justfile, positionalArgs]) => { + const {cwd, args} = getJustContext(justfile, cmd, flags, positionalArgs); + console.log(`-> ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(" ")}`); + return { cwd, args }; + }); + for (const { cwd, args } of invocations) { + if (invokeJust(cwd, args) !== 0) { + return 1; } - return 0; } - return invokeJust(justfile, cmd, flags, positionalArgs); + return 0; } function getJustContext(justfile: string, cmd: string, flags: string[], positionalArgs: string[]): {args: string[], cwd?: string} { @@ -109,11 +104,9 @@ function getJustContext(justfile: string, cmd: string, flags: string[], position } } -function invokeJust(justfile: string, cmd: string, flags: string[], positionalArgs: string[]): number { - const { cwd, args } = getJustContext(justfile, cmd, flags, positionalArgs); - console.log(`-> ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(" ")}`); +function invokeJust(cwd: string | undefined, args: string[]): number { try { - child_process.execFileSync(just, args, { + child_process.execFileSync(vars.just, args, { stdio: "inherit", cwd, }); diff --git a/misc/just/lib.just b/misc/just/lib.just index 94ee77efaa50..5dd5f9b856ef 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -10,6 +10,7 @@ error := style("error") + "error" + NORMAL + ": " cmd_sep := "\n#--------------------------------------------------------\n" export CMD_BEGIN := style("command") + cmd_sep export CMD_END := cmd_sep + NORMAL +export JUST_ERROR := error tsx := "npx tsx@4.19.0" From f2a6503efe83e2ba3ed0a5d83cb5415c791be31d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 18 Jul 2025 14:26:48 +0200 Subject: [PATCH 22/23] Just: use bazel directly for dist building --- misc/just/lib.just | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/just/lib.just b/misc/just/lib.just index 5dd5f9b856ef..3379ab8ed37a 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -29,7 +29,7 @@ _build_dist LANGUAGE: _require_semmle_code (_maybe_build_dist LANGUAGE) [no-exit-message] _maybe_build_dist LANGUAGE: - {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} + {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; tools/bazel run //language-packs:intree-' + LANGUAGE } }}{{ cmd_sep }} default_db_checks := """\ From 0e3ee6efd714292e8f33aebf6d64a9797c27b82b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 18 Jul 2025 15:27:12 +0200 Subject: [PATCH 23/23] Just: reorganize code and revamp formatting --- java/ql/justfile | 2 +- misc/codegen/justfile | 2 +- misc/just/build.just | 12 ++++++ misc/just/defs.just | 58 +++++++++++++++++++++++++++++ misc/just/format.just | 20 ++++++++++ misc/just/lib.just | 54 +-------------------------- rust/ql/integration-tests/justfile | 2 +- rust/ql/justfile | 2 +- swift/justfile | 9 +---- swift/ql/integration-tests/justfile | 2 +- swift/ql/justfile | 2 +- 11 files changed, 100 insertions(+), 65 deletions(-) create mode 100644 misc/just/build.just create mode 100644 misc/just/defs.just create mode 100644 misc/just/format.just diff --git a/java/ql/justfile b/java/ql/justfile index 7a46396fb856..ff0e4c1090f3 100644 --- a/java/ql/justfile +++ b/java/ql/justfile @@ -1,6 +1,6 @@ import "../../lib.just" [no-cd] -format *ARGS=".": (_ql_format ARGS) +format *ARGS=".": (_format_ql ARGS) consistency_queries := source_dir() / "consistency-queries" diff --git a/misc/codegen/justfile b/misc/codegen/justfile index 9dd2019b5fac..8c1e8cdbab49 100644 --- a/misc/codegen/justfile +++ b/misc/codegen/justfile @@ -2,4 +2,4 @@ import "../just/lib.just" test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...") -format *ARGS=".": (_black ARGS) \ No newline at end of file +format *ARGS=".": (_format_py ARGS) diff --git a/misc/just/build.just b/misc/just/build.just new file mode 100644 index 000000000000..226b52a76d3b --- /dev/null +++ b/misc/just/build.just @@ -0,0 +1,12 @@ +import "defs.just" + +_build_dist LANGUAGE: _require_semmle_code (_maybe_build_dist LANGUAGE) + +[no-exit-message] +_maybe_build_dist LANGUAGE: (_if_in_semmle_code ('cd "$SEMMLE_CODE"; tools/bazel run //language-packs:intree-' + LANGUAGE) '# using codeql from PATH, if any') + +[no-cd, no-exit-message] +_bazel COMMAND *ARGS: (_if_in_semmle_code 'cd "$SEMMLE_CODE"; tools/bazel' 'bazel' COMMAND ARGS) + +[no-cd, no-exit-message] +_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS) diff --git a/misc/just/defs.just b/misc/just/defs.just new file mode 100644 index 000000000000..6f46b84597a9 --- /dev/null +++ b/misc/just/defs.just @@ -0,0 +1,58 @@ +import? '../../../semmle-code.just' # internal repo just file, if present +import 'semmle-code-stub.just' + +set fallback +set allow-duplicate-recipes +set allow-duplicate-variables +set unstable + +export PATH_SEP := if os() == "windows" { ";" } else { ":" } +export JUST_EXECUTABLE := just_executable() + +error := style("error") + "error" + NORMAL + ": " +cmd_sep := "\n#--------------------------------------------------------\n" +export CMD_BEGIN := style("command") + cmd_sep +export CMD_END := cmd_sep + NORMAL +export JUST_ERROR := error + +tsx := "npx tsx@4.19.0" + +default_db_checks := """\ + --check-databases \ + --check-diff-informed \ + --fail-on-trap-errors \ +""" + +[no-exit-message] +@_require_semmle_code: + {{ if SEMMLE_CODE == "" { ''' + echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2 + exit 1 + ''' } else { "" } }} + +[no-cd] +_run +ARGS: + {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }} + +[no-cd] +_run_in DIR +ARGS: + {{ cmd_sep }}cd "{{ DIR }}"; {{ ARGS }}{{ cmd_sep }} + +[no-cd] +_run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS) + +[no-cd, positional-arguments, no-exit-message] +@_just +ARGS: + echo "-> just $@" + "{{ JUST_EXECUTABLE }}" "$@" + +[no-cd, positional-arguments] +@_if_not_on_ci_just +ARGS: + if [ "${GITHUB_ACTIONS:-}" != "true" ]; then \ + echo "-> just $@"; \ + "$JUST_EXECUTABLE" "$@"; \ + fi + +[no-cd, no-exit-message] +_if_in_semmle_code THEN ELSE *ARGS: + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { THEN } else { ELSE } }} {{ ARGS }}{{ cmd_sep }} diff --git a/misc/just/format.just b/misc/just/format.just new file mode 100644 index 000000000000..9430cb4c0a94 --- /dev/null +++ b/misc/just/format.just @@ -0,0 +1,20 @@ +import "build.just" + +[no-cd, no-exit-message] +_format_ql +ARGS: (_maybe_build_dist "nolang") ( + _if_in_semmle_code + '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' + 'codeql' + ("query format --in-place -v $(find " + ARGS + " -type f -name '*.ql' -or -name '*.qll')") +) + +[no-cd, no-exit-message] +_format_py *ARGS=".": (_if_in_semmle_code "uv run black" "black" ARGS) + +[no-cd, no-exit-message] +_format_cpp *ARGS=".": ( + _if_in_semmle_code + "uv run clang-format" + "clang-format" + "-i --verbose $(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')" +) diff --git a/misc/just/lib.just b/misc/just/lib.just index 3379ab8ed37a..7fd3156ba467 100644 --- a/misc/just/lib.just +++ b/misc/just/lib.just @@ -1,42 +1,5 @@ -set fallback -set allow-duplicate-recipes -set allow-duplicate-variables -set unstable - -export PATH_SEP := if os() == "windows" { ";" } else { ":" } -export JUST_EXECUTABLE := just_executable() - -error := style("error") + "error" + NORMAL + ": " -cmd_sep := "\n#--------------------------------------------------------\n" -export CMD_BEGIN := style("command") + cmd_sep -export CMD_END := cmd_sep + NORMAL -export JUST_ERROR := error - -tsx := "npx tsx@4.19.0" - -import? '../../../semmle-code.just' # internal repo just file, if present -import 'semmle-code-stub.just' - - -[no-exit-message] -@_require_semmle_code: - {{ if SEMMLE_CODE == "" { ''' - echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2 - exit 1 - ''' } else { "" } }} - -_build_dist LANGUAGE: _require_semmle_code (_maybe_build_dist LANGUAGE) - -[no-exit-message] -_maybe_build_dist LANGUAGE: - {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; tools/bazel run //language-packs:intree-' + LANGUAGE } }}{{ cmd_sep }} - - -default_db_checks := """\ - --check-databases \ - --check-diff-informed \ - --fail-on-trap-errors \ -""" +import "build.just" +import "format.just" [no-cd, positional-arguments, no-exit-message] @_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS: @@ -46,17 +9,7 @@ default_db_checks := """\ @_language_tests EXTRA_ARGS SOURCE_DIR +ROOTS: _require_semmle_code {{ tsx }} "{{ source_dir() }}/language-tests.ts" "$@" -[no-cd, no-exit-message] -_ql_format +ARGS: (_maybe_build_dist "nolang") - {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} - - -[no-cd, no-exit-message] -_bazel COMMAND *ARGS: - {{ cmd_sep }}{{ if SEMMLE_CODE != "" { 'cd "$SEMMLE_CODE"; tools/bazel' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} -[no-cd, no-exit-message] -_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS) [no-cd, no-exit-message] _integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS) @@ -83,6 +36,3 @@ _run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS) echo "-> just $@"; \ "$JUST_EXECUTABLE" "$@"; \ fi - -[no-cd] -_black *ARGS=".": (_run "uv" "run" "black" ARGS) diff --git a/rust/ql/integration-tests/justfile b/rust/ql/integration-tests/justfile index 7f5aab6c508c..ee967d781e63 100644 --- a/rust/ql/integration-tests/justfile +++ b/rust/ql/integration-tests/justfile @@ -7,4 +7,4 @@ test *ARGS=".": (_if_not_on_ci_just "generate" source_dir()) (_integration_test # TODO in separate PR # [no-cd] -# format *ARGS=".": (_ql_format ARGS) (_black ARGS) +# format *ARGS=".": (_format_ql ARGS) (_format_py ARGS) diff --git a/rust/ql/justfile b/rust/ql/justfile index 7a46396fb856..ff0e4c1090f3 100644 --- a/rust/ql/justfile +++ b/rust/ql/justfile @@ -1,6 +1,6 @@ import "../../lib.just" [no-cd] -format *ARGS=".": (_ql_format ARGS) +format *ARGS=".": (_format_ql ARGS) consistency_queries := source_dir() / "consistency-queries" diff --git a/swift/justfile b/swift/justfile index f09fea32e525..72fa15c81563 100644 --- a/swift/justfile +++ b/swift/justfile @@ -1,4 +1,5 @@ import '../lib.just' +import "../../ql/swift/ql/justfile" install: (_bazel "run" "@codeql//swift:install") @@ -7,13 +8,7 @@ build: (_build_dist "swift") generate: (_bazel "run" "@codeql//swift/codegen") -@_check_clang_format: - if ! which clang-format > /dev/null; then \ - "{{ JUST_EXECUTABLE }}" _run_in_semmle_code "tools/bazel" "run" "//c/clang-format:install"; \ - fi - -format ARGS=".": _check_clang_format (_run "clang-format" "-i" ("$(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')")) -import "../../ql/swift/ql/justfile" +format ARGS=".": (_format_cpp ARGS) [group('test')] language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test') diff --git a/swift/ql/integration-tests/justfile b/swift/ql/integration-tests/justfile index 370f7ef87794..dabd36eebd1e 100644 --- a/swift/ql/integration-tests/justfile +++ b/swift/ql/integration-tests/justfile @@ -6,4 +6,4 @@ test *ARGS=".": (_just "generate") (_integration_test ARGS) # TODO in separate PR # [no-cd] -# format *ARGS=".": (_ql_format ARGS) (_black ARGS) +# format *ARGS=".": (_format_ql ARGS) (_format_py ARGS) diff --git a/swift/ql/justfile b/swift/ql/justfile index 7a46396fb856..ff0e4c1090f3 100644 --- a/swift/ql/justfile +++ b/swift/ql/justfile @@ -1,6 +1,6 @@ import "../../lib.just" [no-cd] -format *ARGS=".": (_ql_format ARGS) +format *ARGS=".": (_format_ql ARGS) consistency_queries := source_dir() / "consistency-queries"