Just: introduce common "verbs"

java/justfile

@@ -0,0 +1,3 @@
+import '../lib.just'
+
+build: (_build_dist "java")

java/ql/justfile

@@ -0,0 +1,6 @@
+import "../../lib.just"
+
+[no-cd]
+format *ARGS=".": (_ql_format ARGS)
+
+consistency_queries := source_dir() / "consistency-queries"

java/ql/test/justfile

@@ -0,0 +1,16 @@
+import "../justfile"
+
+base_flags := """\
+     CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \
+"""
+
+all_checks := default_db_checks + """\
+    --check-undefined-labels \
+    --check-repeated-labels \
+    --check-redefined-labels \
+    --check-use-before-definition \
+    --consistency-queries=""" + consistency_queries
+
+[no-cd]
+test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS)
+

justfile

@@ -0,0 +1,2 @@
+import 'misc/just/lib.just'
+import 'misc/just/forward.just'

lib.just

@@ -0,0 +1 @@
+import "misc/just/lib.just"

misc/bazel/justfile

@@ -0,0 +1,5 @@
+import '../just/lib.just'
+
+[no-cd, positional-arguments, no-exit-message]
+hello +ARGS:
+    @echo "hello from bzl" "$@"

misc/codegen/justfile

@@ -0,0 +1,5 @@
+import "../just/lib.just"
+
+test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...")
+
+format *ARGS=".": (_black ARGS)

misc/just/codeql-test-run.ts

@@ -0,0 +1,142 @@
+import * as child_process from "child_process";
+import * as path from "path";
+import * as os from "os";
+import * as fs from "fs";
+
+function invoke(
+    invocation: string[],
+    options: { cwd?: string; log_prefix?: string } = {},
+): number {
+    const log_prefix =
+        options.log_prefix && options.log_prefix !== ""
+            ? `${options.log_prefix} `
+            : "";
+    console.log(
+        `${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`,
+    );
+    try {
+        child_process.execFileSync(invocation[0], invocation.slice(1), {
+            stdio: "inherit",
+            cwd: options.cwd,
+        });
+    } catch (error) {
+        return 1;
+    }
+    return 0;
+}
+
+type Args = {
+    tests: string[];
+    flags: string[];
+    env: string[];
+    codeql: string;
+    all: boolean;
+};
+
+function parseArgs(args: Args, argv: string) {
+    argv.split(/(?<!\\) /)
+        .map((arg) => arg.replace("\\ ", " "))
+        .forEach((arg) => {
+            if (arg.startsWith("--codeql=")) {
+                args.codeql = arg.split("=")[1];
+            } else if (arg === "+" || arg === "--all-checks") {
+                args.all = true;
+            } else if (arg.startsWith("-")) {
+                args.flags.push(arg);
+            } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) {
+                args.env.push(arg);
+            } else if (arg !== "") {
+                args.tests.push(arg);
+            }
+        });
+}
+
+function codeqlTestRun(argv: string[]): number {
+    const semmle_code = process.env["SEMMLE_CODE"];
+    const [language, base_args, all_args, extra_args] = argv;
+    const ram_per_thread = process.platform === "linux" ? 3000 : 2048;
+    const cpus = os.cpus().length;
+    let args: Args = {
+        tests: [],
+        flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`],
+        env: [],
+        codeql: semmle_code ? "build" : "host",
+        all: false,
+    };
+    parseArgs(args, base_args);
+    parseArgs(args, extra_args);
+    if (args.all) {
+        parseArgs(args, all_args);
+    }
+    if (!semmle_code && (args.codeql === "build" || args.codeql === "built")) {
+        console.error(
+            "Using `--codeql=build` or `--codeql=built` requires working with the internal repository",
+        );
+        return 1;
+    }
+    if (args.tests.length === 0) {
+        args.tests.push(".");
+    }
+    if (args.codeql === "build") {
+        if (
+            invoke([process.env["JUST_EXECUTABLE"] || "just", language, "build"], {
+                cwd: semmle_code,
+            }) !== 0
+        ) {
+            return 1;
+        }
+    }
+    if (args.codeql !== "host") {
+        // disable the default implicit config file, but keep an explicit one
+        // this is the same behavior wrt to `--codeql` as the integration test runner
+        process.env["CODEQL_CONFIG_FILE"] ||= ".";
+    }
+    // Set and unset environment variables
+    args.env.forEach((envVar) => {
+        const [key, value] = envVar.split("=", 2);
+        if (key) {
+            if (value === undefined) {
+                delete process.env[key];
+            } else {
+                process.env[key] = value;
+            }
+        } else {
+            console.error(`Invalid environment variable assignment: ${envVar}`);
+            process.exit(1);
+        }
+    });
+    let codeql;
+    if (args.codeql === "built" || args.codeql === "build") {
+        codeql = path.join(
+            semmle_code!,
+            "target",
+            "intree",
+            `codeql-${language}`,
+            "codeql",
+        );
+        if (!fs.existsSync(codeql)) {
+            console.error(`CodeQL executable not found: ${codeql}`);
+            return 1;
+        }
+    } else if (args.codeql === "host") {
+        codeql = "codeql";
+    } else {
+        codeql = args.codeql;
+        if (fs.lstatSync(codeql).isDirectory()) {
+            codeql = path.join(codeql, "codeql");
+            if (process.platform === "win32") {
+                codeql += ".exe";
+            }
+        }
+        if (!fs.existsSync(codeql)) {
+            console.error(`CodeQL executable not found: ${codeql}`);
+            return 1;
+        }
+    }
+
+    return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {
+        log_prefix: args.env.join(" "),
+    });
+}
+
+process.exit(codeqlTestRun(process.argv.slice(2)));

misc/just/forward-command.ts

@@ -0,0 +1,126 @@
+import * as child_process from "child_process";
+import * as path from "path";
+
+const just = process.env["JUST_EXECUTABLE"]!;
+
+console.debug = (...args: any[]) => {}  // comment out to debug script
+
+function checkJustCommand(justfile: string, command: string, postitionalArgs: string[]): boolean {
+    let {cwd, args} = getJustContext(justfile, command, [], postitionalArgs);
+    console.debug(`Checking: ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(", ")}`);
+    const res = child_process.spawnSync(just, ["--dry-run", ...args], {
+        stdio: ["ignore", "ignore", "pipe"],
+        encoding: "utf8",
+        cwd,
+    });
+    console.debug("result:", res);
+    // avoid having the forwarder find itself
+    return res.status === 0 && !res.stderr.includes(`forward-command.ts" ${command} "$@"`);
+}
+
+function commonPath(paths: string[]): string {
+    if (paths.length === 0) return "";
+    if (paths.length === 1) return paths[0];
+    const splitPaths = paths.map((p) => p.split(path.sep));
+    let i;
+    for (i = 0; i <= splitPaths[0].length; i++) {
+        if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) {
+            break;
+        }
+    }
+    return splitPaths[0].slice(0, i).join(path.sep);
+}
+
+function findJustfile(command: string, paths: string[]): string | undefined {
+    const common = commonPath(paths);
+    for (let p = common;; p = path.dirname(p)) {
+        const candidate = path.join(p, "justfile");
+        if (checkJustCommand(candidate, command, paths)) {
+            return candidate;
+        }
+        if (p === "/" || p === ".") {
+            return undefined;
+        }
+    }
+}
+
+function forwardCommand(args: string[]): number {
+    if (args.length == 0) {
+        console.error("No command provided");
+        return 1;
+    }
+    return forward(args[0], args.slice(1));
+}
+
+function forward(cmd: string, args: string[]): number {
+    // non-positional arguments are flags, + (used by language tests) or environment variable settings
+    const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/;
+    const flags = args.filter((arg) => is_non_positional.test(arg));
+    const positionalArgs = args.filter(
+        (arg) => !is_non_positional.test(arg),
+    );
+
+    const justfile = findJustfile(cmd, positionalArgs.length !== 0 ? positionalArgs : ["."]);
+    if (!justfile) {
+        if (positionalArgs.length <= 1) {
+            console.error(`No justfile found for ${cmd}${positionalArgs.length === 0 ? "" : " on " + positionalArgs[0]}`);
+            return 1;
+        }
+        console.log(`no common justfile recipe found for ${cmd} for all arguments, trying one argument at a time`);
+        const runs: [string, string | undefined][] = positionalArgs.map(arg => [arg, findJustfile(cmd, [arg])]);
+        for (const [arg, justfile] of runs) {
+            if (!justfile) {
+                console.error(`No justfile found for ${cmd} on ${arg}`);
+                return 1;
+            }
+        }
+        for (const [arg, justfile] of runs) {
+            if (invokeJust(justfile!, cmd, flags, [arg]) !== 0) {
+                return 1;
+            }
+        }
+        return 0;
+    }
+    return invokeJust(justfile, cmd, flags, positionalArgs);
+}
+
+function getJustContext(justfile: string, cmd: string, flags: string[], positionalArgs: string[]): {args: string[], cwd?: string} {
+    if (positionalArgs.length === 1 && justfile == path.join(positionalArgs[0], "justfile")) {
+        // If there's only one positional argument and it matches the justfile path, suppress arguments
+        // so for example `just build ql/rust` becomes `just build` in the `ql/rust` directory
+        return {
+            cwd: positionalArgs[0],
+            args: [
+                cmd,
+                ...flags,
+            ],
+        };
+    } else {
+        return {
+            cwd: undefined,
+            args: [
+                "--justfile",
+                justfile,
+                cmd,
+                ...flags,
+                ...positionalArgs,
+            ],
+        };
+    }
+}
+
+function invokeJust(justfile: string, cmd: string, flags: string[], positionalArgs: string[]): number {
+    const { cwd, args } = getJustContext(justfile, cmd, flags, positionalArgs);
+    console.log(`-> ${cwd ? `cd ${cwd}; ` : ""}just ${args.join(" ")}`);
+    try {
+        child_process.execFileSync(just, args, {
+            stdio: "inherit",
+            cwd,
+        });
+    } catch (error) {
+        return 1;
+    }
+    return 0;
+}
+
+process.exit(forwardCommand(process.argv.slice(2)));

misc/just/forward.just

@@ -0,0 +1,34 @@
+import "lib.just"
+
+# copy&paste necessary for each command until proper forwarding of multiple args is implemented
+# see https://github.com/casey/just/issues/1988
+
+_forward := tsx + ' "' + source_dir() + '/forward-command.ts"'
+
+alias t := test
+alias b := build
+alias g := generate
+alias gen := generate
+alias f := format
+alias l := lint
+
+[no-cd, positional-arguments, no-exit-message]
+@test *ARGS:
+    {{ _forward }} test "$@"
+
+
+[no-cd, positional-arguments, no-exit-message]
+@build *ARGS:
+    {{ _forward }} build "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@generate *ARGS:
+    {{ _forward }} generate "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@lint *ARGS:
+    {{ _forward }} lint "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@format *ARGS:
+    {{ _forward }} format "$@"

misc/just/justfile

@@ -0,0 +1,2 @@
+format *ARGS=".":
+    npx prettier --write {{ ARGS }}

misc/just/language-tests.ts

@@ -0,0 +1,31 @@
+import * as path from "path";
+import * as process from "process"
+import * as child_process from "child_process";
+
+function languageTests(argv: string[]): number {
+    const [extra_args, dir, ...relativeRoots] = argv;
+    const semmle_code = process.env["SEMMLE_CODE"]!;
+    let roots = relativeRoots.map((root) => path.relative(semmle_code, path.join(dir, root)));
+    const invocation = [
+        process.env["JUST_EXECUTABLE"] || "just",
+        "--justfile",
+        path.join(roots[0], "justfile"),
+        "test",
+        "--all-checks",
+        "--codeql=built",
+        ...extra_args.split(" "),
+        ...roots,
+    ];
+    console.log(`-> just ${invocation.slice(1).join(" ")}`);
+    try {
+        child_process.execFileSync(invocation[0], invocation.slice(1), {
+            stdio: "inherit",
+            cwd: semmle_code,
+        });
+    } catch (error) {
+        return 1;
+    }
+    return 0;
+}
+
+process.exit(languageTests(process.argv.slice(2)));