diff --git a/.gitignore b/.gitignore index 138e488..d50f1ce 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ dist/ build/ # Node -node_modules/ +node_modules # Virtual environments venv/ diff --git a/README.md b/README.md index 25ec446..b9295bb 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ tooling/ Maintained by **Commonalities Working Group**. -To add or modify a CAMARA Validation check, start with the [contributor guide](validation/docs/contributor-guide.md) and the [architecture overview](validation/docs/architecture-overview.md). +To add or modify a CAMARA Validation check, start with the [contributor guide](validation/docs/contributor-guide.md) and the [architecture overview](validation/docs/architecture-overview.md). To run CAMARA Validation against a local API repository clone, see [Running validation locally](validation/docs/contributor-guide.md#running-validation-locally). * Meetings of the working group are held virtually * Schedule: see [Commonalities Working Group wiki page](https://lf-camaraproject.atlassian.net/wiki/x/_QPe) diff --git a/validation/docs/contributor-guide.md b/validation/docs/contributor-guide.md index f1279d0..387c274 100644 --- a/validation/docs/contributor-guide.md +++ b/validation/docs/contributor-guide.md @@ -82,6 +82,32 @@ step (2–4 above), but the check logic itself lives in that engine's native con (`linting/config/.spectral-r4.yaml`, `.gplintrc`, `.yamllint.yaml`) or adapter (`validation/engines/*_adapter.py`), not in `python_checks/`. +## Running validation locally + +To see the full verdict for an API repository before opening a PR, run the orchestrator +against a local clone: + +``` +pip install -r requirements.txt +(cd validation && npm ci) +python3 validation/scripts/validate_local.py [--out ] +``` + +The script validates the clone's checked-out branch with this tooling checkout, as a +`workflow_dispatch` run, and enables validation regardless of +`config/validation-settings.yaml`. The ruleset follows the repository's +`release-plan.yaml`, as in CI. It prints the verdict and the findings per file, and leaves +`summary.json`, `findings.json` and `findings.tsv` in `/diagnostics/`. Exit code 0 is +pass or advisory, 1 is fail, 2 is an error. + +Node tools (Spectral, gplint, Redocly) and the pinned `js-yaml` are read from +`validation/node_modules`. To use an install elsewhere, for example one shared by several +worktrees, set `CAMARA_NODE_MODULES` to that `node_modules` directory; the tests resolve +it the same way. + +The PR check remains the authoritative result: a local run has no PR context, so rules that +depend on the base branch or a Release Review PR do not fire. + ## Regression testing Unit tests verify one check in isolation. Regression testing verifies the framework's diff --git a/validation/engines/node_tools.py b/validation/engines/node_tools.py new file mode 100644 index 0000000..f5d61e5 --- /dev/null +++ b/validation/engines/node_tools.py @@ -0,0 +1,52 @@ +"""Locate the Node tool install (Spectral, Redocly) used by tests and local runs. + +Resolution order for the ``node_modules`` directory: + +1. ``CAMARA_NODE_MODULES`` environment variable +2. a ``spectral`` executable on ``PATH`` that sits in a ``node_modules/.bin`` directory +3. ``validation/node_modules`` in this repository +""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +ENV_VAR = "CAMARA_NODE_MODULES" + +_REPO_DEFAULT = Path(__file__).resolve().parent.parent / "node_modules" + + +def node_modules_dir() -> Path: + """Return the ``node_modules`` directory to use.""" + configured = os.environ.get(ENV_VAR, "").strip() + if configured: + return Path(configured) + + on_path = shutil.which("spectral") + if on_path: + bin_dir = Path(on_path).parent + if bin_dir.name == ".bin": + return bin_dir.parent + + return _REPO_DEFAULT + + +def bin_dir() -> Path: + """Return the ``.bin`` directory of the resolved install.""" + return node_modules_dir() / ".bin" + + +def spectral_bin() -> Path: + """Return the path of the Spectral executable in the resolved install.""" + return bin_dir() / "spectral" + + +def spectral_env() -> dict[str, str]: + """Return the minimal environment for running Spectral under ``node``.""" + return { + "PATH": os.environ.get("PATH", ""), + "NODE_PATH": str(node_modules_dir()), + "HOME": os.environ.get("HOME", ""), + } diff --git a/validation/engines/python_checks/yaml_parser_conformance_checks.py b/validation/engines/python_checks/yaml_parser_conformance_checks.py index 3e506ee..3d693af 100644 --- a/validation/engines/python_checks/yaml_parser_conformance_checks.py +++ b/validation/engines/python_checks/yaml_parser_conformance_checks.py @@ -7,11 +7,13 @@ from __future__ import annotations import json +import os import subprocess from pathlib import Path from typing import List from validation.context import ValidationContext +from validation.engines import node_tools from ._types import make_finding @@ -68,6 +70,7 @@ def _run_helper(repo_path: Path, spec_file: str) -> list[dict] | dict: result = subprocess.run( ["node", str(_HELPER), spec_file], cwd=repo_path, + env={**os.environ, node_tools.ENV_VAR: str(node_tools.node_modules_dir())}, capture_output=True, text=True, timeout=_TIMEOUT_SECONDS, diff --git a/validation/scripts/README.md b/validation/scripts/README.md index 1110eaf..c5f352b 100644 --- a/validation/scripts/README.md +++ b/validation/scripts/README.md @@ -13,6 +13,16 @@ modify its CLI or exit codes without updating that action. python3 validate-release-plan.py [--check-files] ``` +## `validate_local.py` + +Runs the full orchestrator against a local clone of an API repository, as a +`workflow_dispatch` run of its checked-out branch. Usage, prerequisites and +exit codes: [contributor guide](../docs/contributor-guide.md#running-validation-locally). + +``` +python3 validation/scripts/validate_local.py [--out ] +``` + ## `regression_runner.py` Dispatches the validation framework against `regression/*` branches of a test diff --git a/validation/scripts/check-yaml-parser-conformance.mjs b/validation/scripts/check-yaml-parser-conformance.mjs index 50b3d27..6e642ef 100644 --- a/validation/scripts/check-yaml-parser-conformance.mjs +++ b/validation/scripts/check-yaml-parser-conformance.mjs @@ -1,7 +1,23 @@ #!/usr/bin/env node import fs from "node:fs"; -import { load } from "js-yaml"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +// Load the pinned js-yaml from the validation install (CAMARA_NODE_MODULES or +// validation/node_modules) by its explicit path, never from a node_modules +// higher up the tree. The install may be a symlink. +const nodeModules = path.resolve( + process.env.CAMARA_NODE_MODULES?.trim() || + path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "node_modules"), +); +const jsYamlDir = path.join(nodeModules, "js-yaml"); +if (!fs.existsSync(path.join(jsYamlDir, "package.json"))) { + process.stderr.write(`js-yaml not installed in ${nodeModules}\n`); + process.exit(2); +} +const { load } = createRequire(import.meta.url)(jsYamlDir); function toPositiveInt(value, fallback) { return Number.isInteger(value) && value >= 0 ? value + 1 : fallback; diff --git a/validation/scripts/local-validation-settings.yaml b/validation/scripts/local-validation-settings.yaml new file mode 100644 index 0000000..aa9f709 --- /dev/null +++ b/validation/scripts/local-validation-settings.yaml @@ -0,0 +1,8 @@ +# Settings override for validate_local.py: enables validation for any +# repository, including forks and repos not yet onboarded in +# config/validation-settings.yaml. +version: 1 +defaults: + stage: enabled + pr_profile: standard + release_profile: standard diff --git a/validation/scripts/validate_local.py b/validation/scripts/validate_local.py new file mode 100755 index 0000000..d91bc62 --- /dev/null +++ b/validation/scripts/validate_local.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Run the CAMARA validation orchestrator against a local API repository. + +Validates the repository as a ``workflow_dispatch`` run of its checked-out +branch, using this tooling checkout and a settings override that enables +validation for any repository. The ruleset follows ``release-plan.yaml`` +as in CI. + +Usage: + python3 validation/scripts/validate_local.py [--out ] + +Exit codes: + 0 pass or advisory + 1 fail (blocking findings) + 2 error (usage, orchestrator crash, or an engine that did not run) +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Mapping + +TOOLING_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(TOOLING_ROOT)) + +from validation.engines import node_tools # noqa: E402 +from validation.output.formatting import format_rule_label # noqa: E402 + +CONFIG_OVERRIDE = Path(__file__).resolve().parent / "local-validation-settings.yaml" + +NODE_TOOLS = ("spectral", "gplint", "redocly") + +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_ERROR = 2 + +_ORIGIN_RE = re.compile(r"[:/]([^/:]+/[^/]+?)(?:\.git)?/?$") + + +def _git(repo_path: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo_path), *args], + capture_output=True, + text=True, + ) + return result.stdout.strip() if result.returncode == 0 else "" + + +def repo_name(repo_path: Path) -> str: + """Return ``owner/repo`` from the ``origin`` remote, else ``local/``.""" + match = _ORIGIN_RE.search(_git(repo_path, "remote", "get-url", "origin")) + if match: + return match.group(1) + return f"local/{repo_path.resolve().name}" + + +def missing_node_tools() -> list[str]: + """Return the Node tools absent from the resolved ``node_modules/.bin``.""" + return [tool for tool in NODE_TOOLS if not (node_tools.bin_dir() / tool).exists()] + + +def build_env( + repo_path: Path, + output_dir: Path, + base_env: Mapping[str, str] | None = None, +) -> dict[str, str]: + """Return the orchestrator environment for a dispatch-style local run.""" + env = dict(os.environ if base_env is None else base_env) + name = repo_name(repo_path) + env.update( + { + "PATH": os.pathsep.join(filter(None, [str(node_tools.bin_dir()), env.get("PATH")])), + "NODE_PATH": str(node_tools.node_modules_dir()), + "PYTHONPATH": str(TOOLING_ROOT), + "VALIDATION_REPO_PATH": str(repo_path.resolve()), + "VALIDATION_REPO_NAME": name, + "VALIDATION_REPO_OWNER": name.split("/", 1)[0], + "VALIDATION_REF_NAME": _git(repo_path, "symbolic-ref", "--short", "HEAD"), + "VALIDATION_EVENT_NAME": "workflow_dispatch", + "VALIDATION_TOOLING_PATH": str(TOOLING_ROOT), + "VALIDATION_OUTPUT_DIR": str(output_dir), + "VALIDATION_CONFIG_PATH": str(CONFIG_OVERRIDE), + } + ) + return env + + +def _read_summary(output_dir: Path) -> dict | None: + path = output_dir / "diagnostics" / "summary.json" + if not path.is_file(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def format_report(output_dir: Path) -> str: + """Return the verdict, counts and per-file findings of a finished run.""" + diag = output_dir / "diagnostics" + summary = _read_summary(output_dir) + if summary is None: + return f"no summary.json in {diag} (see the orchestrator log above)" + + counts = summary.get("counts", {}) + lines = [ + f"result: {summary.get('result')} - {summary.get('summary', '')}", + "counts: " + " ".join( + f"{key}={counts.get(key, 0)}" for key in ("errors", "warnings", "hints", "blocking") + ), + ] + + findings = json.loads((diag / "findings.json").read_text(encoding="utf-8")) + by_file: dict[str, list[dict]] = {} + for finding in findings: + by_file.setdefault(finding.get("path") or "(no file)", []).append(finding) + for path in sorted(by_file): + lines += ["", path] + for finding in sorted(by_file[path], key=lambda f: f.get("line") or 0): + lines.append( + f" {finding.get('line') or 0}\t{finding.get('level', '')}" + f"\t{format_rule_label(finding)}\t{finding.get('message', '')}" + ) + + lines += ["", f"diagnostics: {diag}"] + return "\n".join(lines) + + +def exit_code(output_dir: Path, orchestrator_rc: int) -> int: + """Map the orchestrator return code and verdict to the script's exit code.""" + summary = _read_summary(output_dir) + if orchestrator_rc != 0 or summary is None: + return EXIT_ERROR + return {"fail": EXIT_FAIL, "error": EXIT_ERROR}.get(summary.get("result"), EXIT_PASS) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Run CAMARA validation against a local API repository." + ) + parser.add_argument("repo_path", type=Path, help="local clone of the API repository") + parser.add_argument( + "--out", type=Path, help="output directory (default: a new temporary directory)" + ) + args = parser.parse_args(argv) + + repo_path = args.repo_path.resolve() + if not (repo_path / "code" / "API_definitions").is_dir(): + parser.error(f"not an API repository (no code/API_definitions): {repo_path}") + missing = missing_node_tools() + if missing: + print( + f"missing in {node_tools.bin_dir()}: {', '.join(missing)}\n" + f"run `npm ci` in validation/ or set {node_tools.ENV_VAR} " + "to an existing node_modules directory", + file=sys.stderr, + ) + return EXIT_ERROR + output_dir = args.out or Path(tempfile.mkdtemp(prefix="camara-validation-")) + + env = build_env(repo_path, output_dir) + print( + f"validating {env['VALIDATION_REPO_NAME']} " + f"(ref {env['VALIDATION_REF_NAME'] or ''}) with {TOOLING_ROOT}", + file=sys.stderr, + ) + rc = subprocess.run( + [sys.executable, "-m", "validation.orchestrator"], + cwd=TOOLING_ROOT, + env=env, + ).returncode + + print(format_report(output_dir)) + return exit_code(output_dir, rc) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/validation/tests/test_node_tools.py b/validation/tests/test_node_tools.py new file mode 100644 index 0000000..7b9bca7 --- /dev/null +++ b/validation/tests/test_node_tools.py @@ -0,0 +1,82 @@ +"""Tests for Node tool resolution (CAMARA_NODE_MODULES -> PATH -> repo default).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from validation.engines import node_tools + +_REPO_DEFAULT = Path(node_tools.__file__).resolve().parent.parent / "node_modules" + + +def _make_install(root: Path) -> Path: + """Create a fake ``node_modules`` with a ``.bin/spectral`` entry.""" + node_modules = root / "node_modules" + (node_modules / ".bin").mkdir(parents=True) + (node_modules / ".bin" / "spectral").write_text("", encoding="utf-8") + return node_modules + + +class TestNodeModulesDir: + def test_env_variable_wins(self, tmp_path, monkeypatch): + install = _make_install(tmp_path / "env") + other = _make_install(tmp_path / "path") + monkeypatch.setenv("CAMARA_NODE_MODULES", str(install)) + monkeypatch.setenv("PATH", str(other / ".bin")) + + assert node_tools.node_modules_dir() == install + + def test_path_lookup_when_env_unset(self, tmp_path, monkeypatch): + install = _make_install(tmp_path) + (install / ".bin" / "spectral").chmod(0o755) + monkeypatch.delenv("CAMARA_NODE_MODULES", raising=False) + monkeypatch.setenv("PATH", str(install / ".bin")) + + assert node_tools.node_modules_dir() == install + + def test_path_hit_outside_bin_dir_is_ignored(self, tmp_path, monkeypatch): + stray = tmp_path / "usr" / "local" / "sbin" + stray.mkdir(parents=True) + (stray / "spectral").write_text("", encoding="utf-8") + (stray / "spectral").chmod(0o755) + monkeypatch.delenv("CAMARA_NODE_MODULES", raising=False) + monkeypatch.setenv("PATH", str(stray)) + + assert node_tools.node_modules_dir() == _REPO_DEFAULT + + def test_repo_default_when_nothing_else_resolves(self, monkeypatch): + monkeypatch.delenv("CAMARA_NODE_MODULES", raising=False) + monkeypatch.setenv("PATH", "") + + assert node_tools.node_modules_dir() == _REPO_DEFAULT + + def test_blank_env_variable_is_ignored(self, monkeypatch): + monkeypatch.setenv("CAMARA_NODE_MODULES", " ") + monkeypatch.setenv("PATH", "") + + assert node_tools.node_modules_dir() == _REPO_DEFAULT + + +class TestSpectralHelpers: + def test_spectral_bin_is_under_resolved_install(self, tmp_path, monkeypatch): + install = _make_install(tmp_path) + monkeypatch.setenv("CAMARA_NODE_MODULES", str(install)) + + assert node_tools.spectral_bin() == install / ".bin" / "spectral" + + def test_spectral_env_exposes_node_path(self, tmp_path, monkeypatch): + install = _make_install(tmp_path) + monkeypatch.setenv("CAMARA_NODE_MODULES", str(install)) + + env = node_tools.spectral_env() + + assert env["NODE_PATH"] == str(install) + assert set(env) == {"PATH", "NODE_PATH", "HOME"} + + def test_bin_dir_is_under_resolved_install(self, tmp_path, monkeypatch): + install = _make_install(tmp_path) + monkeypatch.setenv("CAMARA_NODE_MODULES", str(install)) + + assert node_tools.bin_dir() == install / ".bin" diff --git a/validation/tests/test_python_checks_bundling.py b/validation/tests/test_python_checks_bundling.py index ad852e8..9ee6f22 100644 --- a/validation/tests/test_python_checks_bundling.py +++ b/validation/tests/test_python_checks_bundling.py @@ -8,10 +8,7 @@ from unittest.mock import Mock from validation.context import ApiContext, ValidationContext - - -_REPO_ROOT = Path(__file__).resolve().parent.parent.parent -_REDOCLY_BIN_DIR = _REPO_ROOT / "validation" / "node_modules" / ".bin" +from validation.engines import node_tools def _make_context(api_name: str | None = None) -> ValidationContext: @@ -340,7 +337,7 @@ class TestComponentRenamingConflictIntegration: def _run_with_real_redocly(self, tmp_path, monkeypatch, api_name): from validation.engines.python_checks import bundling_checks - monkeypatch.setenv("PATH", f"{_REDOCLY_BIN_DIR}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setenv("PATH", f"{node_tools.bin_dir()}{os.pathsep}{os.environ['PATH']}") return bundling_checks.check_component_renaming_conflict( tmp_path, _make_context(api_name) ) diff --git a/validation/tests/test_spectral_gap_rules.py b/validation/tests/test_spectral_gap_rules.py index fc35e68..917b086 100644 --- a/validation/tests/test_spectral_gap_rules.py +++ b/validation/tests/test_spectral_gap_rules.py @@ -16,13 +16,14 @@ import pytest +from validation.engines import node_tools + # --------------------------------------------------------------------------- # Paths & helpers # --------------------------------------------------------------------------- _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _RULESET = _REPO_ROOT / "linting" / "config" / ".spectral-r4.yaml" -_NODE_MODULES = _REPO_ROOT / "validation" / "node_modules" def _run_spectral(yaml_content: str) -> list[dict]: @@ -32,15 +33,11 @@ def _run_spectral(yaml_content: str) -> list[dict]: f.flush() tmp_path = f.name - env = { - "PATH": subprocess.os.environ.get("PATH", ""), - "NODE_PATH": str(_NODE_MODULES), - "HOME": subprocess.os.environ.get("HOME", ""), - } + env = node_tools.spectral_env() result = subprocess.run( [ "node", - str(_NODE_MODULES / ".bin" / "spectral"), + str(node_tools.spectral_bin()), "lint", tmp_path, "-r", str(_RULESET), @@ -925,16 +922,12 @@ def _run_spectral_raw(yaml_content: str) -> subprocess.CompletedProcess: f.write(yaml_content) f.flush() tmp_path = f.name - env = { - "PATH": subprocess.os.environ.get("PATH", ""), - "NODE_PATH": str(_NODE_MODULES), - "HOME": subprocess.os.environ.get("HOME", ""), - } + env = node_tools.spectral_env() try: return subprocess.run( [ "node", - str(_NODE_MODULES / ".bin" / "spectral"), + str(node_tools.spectral_bin()), "lint", tmp_path, "-r", str(_RULESET), "--format", "json", ], capture_output=True, text=True, env=env, timeout=30, diff --git a/validation/tests/test_spectral_s011_components.py b/validation/tests/test_spectral_s011_components.py index 6706ad2..ac9dbbb 100644 --- a/validation/tests/test_spectral_s011_components.py +++ b/validation/tests/test_spectral_s011_components.py @@ -17,6 +17,7 @@ import tempfile from pathlib import Path +from validation.engines import node_tools from validation.engines.spectral_adapter import parse_spectral_output # --------------------------------------------------------------------------- @@ -27,7 +28,6 @@ _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _RULESET = _REPO_ROOT / "linting" / "config" / ".spectral-r4.yaml" -_NODE_MODULES = _REPO_ROOT / "validation" / "node_modules" def _run_spectral(yaml_content: str) -> list[dict]: @@ -36,15 +36,11 @@ def _run_spectral(yaml_content: str) -> list[dict]: f.flush() tmp_path = f.name - env = { - "PATH": subprocess.os.environ.get("PATH", ""), - "NODE_PATH": str(_NODE_MODULES), - "HOME": subprocess.os.environ.get("HOME", ""), - } + env = node_tools.spectral_env() result = subprocess.run( [ "node", - str(_NODE_MODULES / ".bin" / "spectral"), + str(node_tools.spectral_bin()), "lint", tmp_path, "-r", str(_RULESET), @@ -72,15 +68,11 @@ def _run_spectral_multi_file(entry_content: str, sibling_files: dict[str, str]) for name, content in sibling_files.items(): (Path(tmp_dir) / name).write_text(content) - env = { - "PATH": subprocess.os.environ.get("PATH", ""), - "NODE_PATH": str(_NODE_MODULES), - "HOME": subprocess.os.environ.get("HOME", ""), - } + env = node_tools.spectral_env() result = subprocess.run( [ "node", - str(_NODE_MODULES / ".bin" / "spectral"), + str(node_tools.spectral_bin()), "lint", str(entry_path), "-r", str(_RULESET), diff --git a/validation/tests/test_spectral_unused_components.py b/validation/tests/test_spectral_unused_components.py index 8eb1d50..a3a1cac 100644 --- a/validation/tests/test_spectral_unused_components.py +++ b/validation/tests/test_spectral_unused_components.py @@ -3,13 +3,14 @@ from __future__ import annotations import json -import os import subprocess import tempfile from pathlib import Path import pytest +from validation.engines import node_tools + _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _RULESET = _REPO_ROOT / "linting" / "config" / ".spectral-r4.yaml" @@ -18,7 +19,6 @@ _REPO_ROOT / "linting" / "config" / ".spectral-r3.4.yaml", _RULESET, ] -_NODE_MODULES = _REPO_ROOT / "validation" / "node_modules" _RULE = "camara-discriminator-aware-unused-component" @@ -28,16 +28,12 @@ def _run_spectral(spec: str, ruleset: Path = _RULESET) -> list[dict]: file.flush() spec_path = Path(file.name) - env = { - "PATH": os.environ.get("PATH", ""), - "NODE_PATH": str(_NODE_MODULES), - "HOME": os.environ.get("HOME", ""), - } + env = node_tools.spectral_env() try: result = subprocess.run( [ "node", - str(_NODE_MODULES / ".bin" / "spectral"), + str(node_tools.spectral_bin()), "lint", str(spec_path), "-r", diff --git a/validation/tests/test_spectral_x_correlator.py b/validation/tests/test_spectral_x_correlator.py index 6c47708..5108b06 100644 --- a/validation/tests/test_spectral_x_correlator.py +++ b/validation/tests/test_spectral_x_correlator.py @@ -3,15 +3,15 @@ from __future__ import annotations import json -import os import subprocess import tempfile from pathlib import Path +from validation.engines import node_tools + _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _RULESET = _REPO_ROOT / "linting" / "config" / ".spectral-r4.yaml" -_NODE_MODULES = _REPO_ROOT / "validation" / "node_modules" _REQUEST_RULE = "camara-x-correlator-request-parameter" _RESPONSE_RULE = "camara-x-correlator-response-header" @@ -24,15 +24,11 @@ def _run_spectral(files: dict[str, str], entrypoint: str = "api.yaml") -> list[d target.parent.mkdir(parents=True, exist_ok=True) target.write_text(content, encoding="utf-8") - env = { - "PATH": os.environ.get("PATH", ""), - "NODE_PATH": str(_NODE_MODULES), - "HOME": os.environ.get("HOME", ""), - } + env = node_tools.spectral_env() result = subprocess.run( [ "node", - str(_NODE_MODULES / ".bin" / "spectral"), + str(node_tools.spectral_bin()), "lint", str(root / entrypoint), "-r", diff --git a/validation/tests/test_validate_local.py b/validation/tests/test_validate_local.py new file mode 100644 index 0000000..f5cd4ce --- /dev/null +++ b/validation/tests/test_validate_local.py @@ -0,0 +1,214 @@ +"""Tests for the local validation runner (validation/scripts/validate_local.py).""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +# validation/scripts/ is not a package — load the module directly. +_ROOT = Path(__file__).resolve().parents[2] +_MODULE_PATH = _ROOT / "validation" / "scripts" / "validate_local.py" +_spec = importlib.util.spec_from_file_location("validate_local", _MODULE_PATH) +assert _spec is not None and _spec.loader is not None +validate_local = importlib.util.module_from_spec(_spec) +sys.modules["validate_local"] = validate_local +_spec.loader.exec_module(validate_local) + + +def _git(repo: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) + + +def _init_repo(path: Path, origin: str | None = None, branch: str = "main") -> Path: + path.mkdir(parents=True, exist_ok=True) + _git(path, "init", "-q", "-b", branch) + if origin is not None: + _git(path, "remote", "add", "origin", origin) + return path + + +class TestRepoName: + @pytest.mark.parametrize( + "origin", + [ + "https://github.com/camaraproject/ReleaseTest.git", + "https://github.com/camaraproject/ReleaseTest", + "git@github.com:camaraproject/ReleaseTest.git", + ], + ) + def test_owner_and_repo_from_origin(self, tmp_path, origin): + repo = _init_repo(tmp_path / "clone", origin) + + assert validate_local.repo_name(repo) == "camaraproject/ReleaseTest" + + def test_local_fallback_without_origin(self, tmp_path): + repo = _init_repo(tmp_path / "MyApi") + + assert validate_local.repo_name(repo) == "local/MyApi" + + +class TestBuildEnv: + def test_dispatch_run_of_the_checked_out_branch(self, tmp_path): + repo = _init_repo(tmp_path / "api", "https://github.com/camaraproject/ReleaseTest.git", "feature/x") + out = tmp_path / "out" + + env = validate_local.build_env(repo, out, base_env={"PATH": "/usr/bin"}) + + assert env["VALIDATION_REPO_PATH"] == str(repo.resolve()) + assert env["VALIDATION_REPO_NAME"] == "camaraproject/ReleaseTest" + assert env["VALIDATION_REPO_OWNER"] == "camaraproject" + assert env["VALIDATION_REF_NAME"] == "feature/x" + assert env["VALIDATION_EVENT_NAME"] == "workflow_dispatch" + assert env["VALIDATION_TOOLING_PATH"] == str(_ROOT) + assert env["VALIDATION_OUTPUT_DIR"] == str(out) + + def test_node_tools_on_path(self, tmp_path, monkeypatch): + install = tmp_path / "nm" + (install / ".bin").mkdir(parents=True) + monkeypatch.setenv("CAMARA_NODE_MODULES", str(install)) + repo = _init_repo(tmp_path / "api") + + env = validate_local.build_env(repo, tmp_path / "out", base_env={"PATH": "/usr/bin"}) + + assert env["PATH"].split(":")[:2] == [str(install / ".bin"), "/usr/bin"] + assert env["NODE_PATH"] == str(install) + + def test_config_override_passes_the_stage_gate(self, tmp_path): + from validation.config.config_gate import resolve_stage_from_files + + repo = _init_repo(tmp_path / "api") + env = validate_local.build_env(repo, tmp_path / "out", base_env={}) + + result = resolve_stage_from_files( + config_path=Path(env["VALIDATION_CONFIG_PATH"]), + schema_path=_ROOT / "validation" / "schemas" / "validation-settings-schema.yaml", + repo_full_name=env["VALIDATION_REPO_NAME"], + repo_owner=env["VALIDATION_REPO_OWNER"], + trigger_type=env["VALIDATION_EVENT_NAME"], + ) + + assert result.should_continue + assert result.stage == "enabled" + + +def _write_diagnostics(out: Path, result: str, findings: list[dict]) -> Path: + diag = out / "diagnostics" + diag.mkdir(parents=True) + summary = { + "result": result, + "summary": "1 error, 1 warning", + "counts": {"errors": 1, "warnings": 1, "hints": 0, "total": 2, "blocking": 1}, + } + (diag / "summary.json").write_text(json.dumps(summary), encoding="utf-8") + (diag / "findings.json").write_text(json.dumps(findings), encoding="utf-8") + return diag + + +_FINDINGS = [ + {"rule_id": "S-011", "path": "code/API_definitions/b.yaml", "line": 40, + "level": "warn", "message": "late"}, + {"engine_rule": "camara-x", "path": "code/API_definitions/a.yaml", "line": 7, + "level": "error", "message": "first"}, + {"rule_id": "S-002", "path": "code/API_definitions/b.yaml", "line": 3, + "level": "error", "message": "early"}, +] + + +class TestReport: + def test_findings_grouped_by_file_in_line_order(self, tmp_path): + diag = _write_diagnostics(tmp_path, "fail", _FINDINGS) + + lines = validate_local.format_report(tmp_path).splitlines() + + assert lines[0] == "result: fail - 1 error, 1 warning" + assert "errors=1 warnings=1 hints=0 blocking=1" in lines[1] + body = [line for line in lines[2:] if line] + assert body[:5] == [ + "code/API_definitions/a.yaml", + " 7\terror\tcamara-x\tfirst", + "code/API_definitions/b.yaml", + " 3\terror\tS-002\tearly", + " 40\twarn\tS-011\tlate", + ] + assert lines[-1] == f"diagnostics: {diag}" + + def test_missing_summary_is_reported(self, tmp_path): + assert "no summary.json" in validate_local.format_report(tmp_path) + + +class TestExitCode: + @pytest.mark.parametrize( + ("result", "expected"), + [("pass", 0), ("advisory", 0), ("fail", 1), ("error", 2)], + ) + def test_maps_result(self, tmp_path, result, expected): + _write_diagnostics(tmp_path, result, []) + + assert validate_local.exit_code(tmp_path, orchestrator_rc=0) == expected + + def test_orchestrator_crash_wins(self, tmp_path): + _write_diagnostics(tmp_path, "pass", []) + + assert validate_local.exit_code(tmp_path, orchestrator_rc=2) == 2 + + def test_missing_summary_is_an_error(self, tmp_path): + assert validate_local.exit_code(tmp_path, orchestrator_rc=0) == 2 + + +_MINIMAL_SPEC = """\ +openapi: 3.0.3 +info: + title: Sample + version: wip + description: Sample API. +servers: + - url: "{apiRoot}/sample/vwip" + variables: + apiRoot: + default: http://localhost:9091 +paths: {} +""" + + +class TestMain: + def test_rejects_a_directory_without_api_definitions(self, tmp_path, capsys): + with pytest.raises(SystemExit): + validate_local.main([str(tmp_path)]) + + assert "code/API_definitions" in capsys.readouterr().err + + def test_stops_when_the_node_install_is_incomplete(self, tmp_path, monkeypatch, capsys): + install = tmp_path / "nm" + (install / ".bin").mkdir(parents=True) + (install / ".bin" / "spectral").write_text("", encoding="utf-8") + monkeypatch.setenv("CAMARA_NODE_MODULES", str(install)) + repo = _init_repo(tmp_path / "SampleApi") + (repo / "code" / "API_definitions").mkdir(parents=True) + + rc = validate_local.main([str(repo), "--out", str(tmp_path / "out")]) + + err = capsys.readouterr().err + assert rc == validate_local.EXIT_ERROR + assert "gplint, redocly" in err + assert "CAMARA_NODE_MODULES" in err + assert not (tmp_path / "out" / "diagnostics").exists() + + def test_runs_the_orchestrator_and_prints_the_report(self, tmp_path, capsys): + repo = _init_repo(tmp_path / "SampleApi", "https://github.com/camaraproject/SampleApi.git") + api_dir = repo / "code" / "API_definitions" + api_dir.mkdir(parents=True) + (api_dir / "sample.yaml").write_text(_MINIMAL_SPEC, encoding="utf-8") + out = tmp_path / "out" + + rc = validate_local.main([str(repo), "--out", str(out)]) + + report = capsys.readouterr().out + assert (out / "diagnostics" / "summary.json").is_file() + assert rc in (validate_local.EXIT_PASS, validate_local.EXIT_FAIL) + assert report.startswith("result: ") + assert "code/API_definitions/sample.yaml" in report diff --git a/validation/tests/test_yaml_parser_conformance.py b/validation/tests/test_yaml_parser_conformance.py index 19aeb3a..abecf35 100644 --- a/validation/tests/test_yaml_parser_conformance.py +++ b/validation/tests/test_yaml_parser_conformance.py @@ -302,3 +302,58 @@ def test_helper_failure_is_visible_as_error(self, tmp_path: Path, monkeypatch): assert findings[0]["engine_rule"] == "yaml-parser-conformance-execution-error" assert findings[0]["level"] == "error" assert "helper failed" in findings[0]["message"] + + +def _make_fake_js_yaml(root: Path) -> Path: + """Create a ``node_modules`` whose ``js-yaml`` rejects every document.""" + node_modules = root / "node_modules" + package = node_modules / "js-yaml" + package.mkdir(parents=True) + (package / "package.json").write_text('{"name": "js-yaml", "main": "index.js"}', encoding="utf-8") + (package / "index.js").write_text( + 'exports.load = () => { throw new Error("fake js-yaml"); };\n', encoding="utf-8" + ) + (node_modules / ".bin").mkdir() + spectral = node_modules / ".bin" / "spectral" + spectral.write_text("", encoding="utf-8") + spectral.chmod(0o755) + return node_modules + + +class TestNodeModulesResolution: + def test_helper_loads_js_yaml_from_camara_node_modules(self, tmp_path: Path, monkeypatch): + node_modules = _make_fake_js_yaml(tmp_path / "install") + _write_api_definition(tmp_path, "sample", _VALID_OPENAPI) + monkeypatch.setenv("CAMARA_NODE_MODULES", str(node_modules)) + + findings = _run_helper(tmp_path, "code/API_definitions/sample.yaml") + + assert [f["reason"] for f in findings] == ["fake js-yaml"] + + def test_helper_follows_a_symlinked_install(self, tmp_path: Path, monkeypatch): + node_modules = _make_fake_js_yaml(tmp_path / "install") + link = tmp_path / "shared-install" + link.symlink_to(node_modules, target_is_directory=True) + _write_api_definition(tmp_path, "sample", _VALID_OPENAPI) + monkeypatch.setenv("CAMARA_NODE_MODULES", str(link)) + + findings = _run_helper(tmp_path, "code/API_definitions/sample.yaml") + + assert [f["reason"] for f in findings] == ["fake js-yaml"] + + def test_check_passes_the_resolved_install_to_the_helper(self, tmp_path: Path, monkeypatch): + from validation.engines.python_checks import yaml_parser_conformance_checks + + node_modules = _make_fake_js_yaml(tmp_path / "install") + _write_api_definition(tmp_path, "sample", _VALID_OPENAPI) + monkeypatch.delenv("CAMARA_NODE_MODULES", raising=False) + monkeypatch.setenv("PATH", f"{node_modules / '.bin'}:{subprocess.os.environ['PATH']}") + + findings = yaml_parser_conformance_checks.check_yaml_parser_conformance( + tmp_path, + _make_context("sample"), + ) + + assert [f["message"] for f in findings] == [ + "OpenAPI YAML fails parser conformance: fake js-yaml" + ]