From bd9cc082dda39d60b1b30934d6ef71fd65e4d978 Mon Sep 17 00:00:00 2001 From: therealsaitama Date: Fri, 26 Jun 2026 00:16:14 +0530 Subject: [PATCH 1/4] feat: add release readiness workflow --- Makefile | 2 +- README.md | 20 +- scripts/release_readiness.py | 658 +++++++++++++++++++++++++++++++++++ 3 files changed, 676 insertions(+), 4 deletions(-) create mode 100755 scripts/release_readiness.py diff --git a/Makefile b/Makefile index 0458e30..1fdda95 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ smoke: scripts/smoke_install.sh release-check: - $(PYTHON) scripts/check_release_version.py --base $(BASE) + $(PYTHON) scripts/release_readiness.py --base $(BASE) clean: rm -rf $(GEN_ROOT) .pytest_cache .mypy_cache .ruff_cache htmlcov dist build diff --git a/README.md b/README.md index 5052df2..7de196a 100644 --- a/README.md +++ b/README.md @@ -156,16 +156,30 @@ The smoke script validates the repository, compiles Python scripts without writi ## Release discipline -For any change that affects profile behavior, generated files, config, docs, skills, scripts, or distribution metadata: +For any change that affects profile behavior, generated files, config, docs, skills, scripts, or distribution metadata, a release readiness workflow must be run. -1. Bump `version` in `distribution.yaml`. +Before opening a pull request or tagging a release: + +1. Bump the `version` in `distribution.yaml`. 2. Add a matching `## ` entry to `CHANGELOG.md`. -3. Run the release guard before opening a pull request: +3. Run the release readiness check: ```bash make release-check ``` +This runs `scripts/release_readiness.py`, which checks: +- **Version Discipline**: Ensures version changes when release-relevant files are modified (relative to a base branch, e.g., `origin/main`). +- **Changelog Validation**: Verifies that a matching release entry heading exists in `CHANGELOG.md`. +- **Profile Validation**: Runs standard checks from `validate_profile.py` (formatting, skills, metadata). +- **No Runtime Files**: Verifies no runtime, temporary, or cache files (e.g., `.env`, `state.db`) are committed. +- **No Secrets**: Scans the workspace to prevent hardcoded secrets or credentials. +- **Generator Smoke**: Runs a full template compile and verifies the generated profile passes validation. +- **Install Smoke**: If Hermes CLI is available, verifies that the profile installs successfully and contains required files. +- **Install Documentation**: Ensures that the `README.md` or other docs list the correct installation command for this repository. + +It generates a detailed Markdown report summarizing the checks, status (PASS, FAIL, SKIPPED), and any remediation hints. + The GitHub Actions release guard enforces this on pull requests. ## Publish a generated profile diff --git a/scripts/release_readiness.py b/scripts/release_readiness.py new file mode 100755 index 0000000..8d45a0d --- /dev/null +++ b/scripts/release_readiness.py @@ -0,0 +1,658 @@ +#!/usr/bin/env python3 +"""Run release readiness workflow checks and generate a Markdown report.""" +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +try: + import yaml +except ImportError as exc: # pragma: no cover + raise SystemExit("PyYAML is required. Install with: python3 -m pip install pyyaml") from exc + +# Release relevant prefixes (reused from check_release_version.py) +RELEASE_RELEVANT_PREFIXES = ( + "SOUL.md", + "AGENTS.md", + "README.md", + "config.yaml", + "distribution.yaml", + ".env.EXAMPLE", + "mcp.json", + "skills/", + "templates/", + "scripts/", + ".github/workflows/", + "SECURITY.md", + "CONTRIBUTING.md", + "github-repo-metadata.yaml", + "requirements.txt", + "Makefile", + "docs/", +) + +IGNORED_PATHS = { + "CHANGELOG.md", +} + +# Forbidden directories and files (reused from validate_profile.py) +USER_OWNED = { + ".env", + "auth.json", + "state.db", + "state.db-shm", + "state.db-wal", + "memories", + "sessions", + "logs", + "workspace", + "plans", + "local", + "cache", +} + +FORBIDDEN_DIR_NAMES = USER_OWNED | { + "__pycache__", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + "htmlcov", + "dist", + "build", + "hook-sessions", +} + +FORBIDDEN_FILE_NAMES = { + ".coverage", + "coverage.xml", +} + +FORBIDDEN_SUFFIXES = ( + ".pyc", + ".pyo", + ".pyd", +) + +SECRET_PATTERNS = [ + re.compile(r"ghp_[A-Za-z0-9_]{20,}"), + re.compile(r"gho_[A-Za-z0-9_]{20,}"), + re.compile(r"sk-[A-Za-z0-9]{20,}"), + re.compile(r"xox[baprs]-[A-Za-z0-9-]{20,}"), + re.compile(r"AKIA[0-9A-Z]{16}"), +] + + +def run_git(args: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True) + + +def read_current_version(root: Path) -> str: + data = yaml.safe_load((root / "distribution.yaml").read_text(encoding="utf-8")) or {} + version = str(data.get("version") or "").strip() + if not version: + raise ValueError("distribution.yaml missing version") + return version + + +def read_base_version(root: Path, base: str) -> str | None: + proc = run_git(["show", f"{base}:distribution.yaml"], root) + if proc.returncode != 0: + return None + try: + data = yaml.safe_load(proc.stdout) or {} + return str(data.get("version") or "").strip() or None + except Exception: + return None + + +def changed_files(root: Path, base: str) -> list[str] | None: + proc = run_git(["diff", "--name-only", f"{base}...HEAD"], root) + if proc.returncode != 0: + proc = run_git(["diff", "--name-only", base, "HEAD"], root) + if proc.returncode != 0: + return None + return [line.strip() for line in proc.stdout.splitlines() if line.strip()] + + +def is_release_relevant(path: str) -> bool: + if path in IGNORED_PATHS: + return False + return any(path == prefix.rstrip("/") or path.startswith(prefix) for prefix in RELEASE_RELEVANT_PREFIXES) + + +def iter_validation_paths(root: Path) -> list[Path]: + if (root / ".git").exists(): + proc = subprocess.run( + ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], + cwd=root, + text=False, + capture_output=True, + ) + if proc.returncode == 0: + rels = [item.decode("utf-8") for item in proc.stdout.split(b"\0") if item] + return [root / rel for rel in rels] + return [path for path in root.rglob("*") if path.is_file()] + + +def check_version_discipline(root: Path, base: str, strict: bool) -> dict[str, str]: + files = changed_files(root, base) + if files is None: + message = f"Could not compare against base ref {base}" + if strict: + return { + "name": "Version Discipline", + "status": "FAIL", + "description": message, + "hint": f"Ensure base ref '{base}' is fetched and exists locally. Try running 'git fetch origin'." + } + else: + return { + "name": "Version Discipline", + "status": "SKIPPED", + "description": message, + "hint": "Run with a valid base ref to enforce version checking." + } + + relevant = [path for path in files if is_release_relevant(path)] + if not relevant: + return { + "name": "Version Discipline", + "status": "PASS", + "description": "No release-relevant changes detected.", + "hint": "" + } + + try: + current_version = read_current_version(root) + except Exception as exc: + return { + "name": "Version Discipline", + "status": "FAIL", + "description": f"Failed to read current version: {exc}", + "hint": "Check that distribution.yaml exists and has a valid version field." + } + + base_version = read_base_version(root, base) + if base_version is None: + message = f"Could not read distribution.yaml from base ref {base}" + if strict: + return { + "name": "Version Discipline", + "status": "FAIL", + "description": message, + "hint": "Ensure distribution.yaml exists in the base branch and contains a valid version." + } + else: + return { + "name": "Version Discipline", + "status": "SKIPPED", + "description": message, + "hint": "Base branch distribution.yaml is missing or invalid. Check skipped." + } + + if current_version == base_version: + return { + "name": "Version Discipline", + "status": "FAIL", + "description": f"distribution.yaml version did not change from {base_version}.", + "hint": f"Increment the 'version' field in distribution.yaml (currently {current_version})." + } + + return { + "name": "Version Discipline", + "status": "PASS", + "description": f"Version bumped from {base_version} to {current_version}.", + "hint": "" + } + + +def check_changelog_heading(root: Path) -> dict[str, str]: + try: + current_version = read_current_version(root) + except Exception as exc: + return { + "name": "Changelog Heading", + "status": "FAIL", + "description": f"Cannot determine current version: {exc}", + "hint": "Fix distribution.yaml version first." + } + + changelog_path = root / "CHANGELOG.md" + if not changelog_path.exists(): + return { + "name": "Changelog Heading", + "status": "FAIL", + "description": "CHANGELOG.md does not exist.", + "hint": "Create CHANGELOG.md and add release notes." + } + + text = changelog_path.read_text(encoding="utf-8") + pattern = rf"^##\s+\[?{re.escape(current_version)}\]?\b" + if re.search(pattern, text, flags=re.MULTILINE) is not None: + return { + "name": "Changelog Heading", + "status": "PASS", + "description": f"Found matching entry '## {current_version}' in CHANGELOG.md.", + "hint": "" + } + + return { + "name": "Changelog Heading", + "status": "FAIL", + "description": f"Missing entry for version {current_version} in CHANGELOG.md.", + "hint": f"Add a heading like '## {current_version}' to CHANGELOG.md and document the changes." + } + + +def check_profile_validation(root: Path) -> dict[str, str]: + validator_script = root / "scripts" / "validate_profile.py" + if not validator_script.exists(): + return { + "name": "Profile Validation", + "status": "FAIL", + "description": "scripts/validate_profile.py does not exist.", + "hint": "Restore scripts/validate_profile.py in the repository." + } + + proc = subprocess.run([sys.executable, str(validator_script), str(root)], capture_output=True, text=True) + if proc.returncode == 0: + return { + "name": "Profile Validation", + "status": "PASS", + "description": "Profile validation checks passed.", + "hint": "" + } + + # Extract errors starting with ERROR: + errors = [line.strip() for line in proc.stdout.splitlines() if line.startswith("ERROR:")] + if not errors: + errors = [proc.stderr.strip() or proc.stdout.strip()] + + return { + "name": "Profile Validation", + "status": "FAIL", + "description": f"Validation failed with {len(errors)} error(s).", + "hint": "
".join(errors) or "Run 'make validate' to see errors." + } + + +def check_runtime_files(root: Path) -> dict[str, str]: + seen_dirs: set[Path] = set() + forbidden: list[str] = [] + + for path in iter_validation_paths(root): + if ".git" in path.parts: + continue + rel = path.relative_to(root) + for parent in rel.parents: + if str(parent) == "." or parent in seen_dirs: + continue + seen_dirs.add(parent) + if parent.name in FORBIDDEN_DIR_NAMES: + forbidden.append(f"Directory '{parent}'") + if path.name in FORBIDDEN_FILE_NAMES: + forbidden.append(f"File '{rel}'") + if path.name in USER_OWNED: + forbidden.append(f"User-owned file '{rel}'") + if path.name.endswith(FORBIDDEN_SUFFIXES): + forbidden.append(f"Cache file '{rel}'") + + if forbidden: + forbidden = sorted(list(set(forbidden))) + return { + "name": "No Runtime Files", + "status": "FAIL", + "description": f"Found forbidden runtime files or caches in repository: {', '.join(forbidden[:5])}" + ("..." if len(forbidden) > 5 else ""), + "hint": "Delete these files or add them to your .gitignore/exclude rules so they are not tracked or committed." + } + + return { + "name": "No Runtime Files", + "status": "PASS", + "description": "No runtime files, user-owned files, or caches detected.", + "hint": "" + } + + +def check_secrets(root: Path) -> dict[str, str]: + skip_dirs = {".git", "node_modules", ".venv", "venv", "__pycache__"} + flagged_files: list[str] = [] + + for path in root.rglob("*"): + if not path.is_file() or any(part in skip_dirs for part in path.parts): + continue + if path.suffix.lower() in {".png", ".jpg", ".jpeg", ".gif", ".pdf"}: + continue + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + for pattern in SECRET_PATTERNS: + if pattern.search(text): + flagged_files.append(str(path.relative_to(root))) + break + + if flagged_files: + return { + "name": "No Secrets", + "status": "FAIL", + "description": f"Possible credentials or secrets detected in: {', '.join(flagged_files[:5])}" + ("..." if len(flagged_files) > 5 else ""), + "hint": "Remove any private keys, passwords, API tokens, or secrets from files in the repository." + } + + return { + "name": "No Secrets", + "status": "PASS", + "description": "No hardcoded secrets or credentials detected.", + "hint": "" + } + + +def check_generator_smoke(root: Path) -> dict[str, str]: + gen_script = root / "scripts" / "generate_profile.py" + params_file = root / "templates" / "profile.params.yaml" + + if not gen_script.exists(): + return { + "name": "Generator Smoke", + "status": "FAIL", + "description": "scripts/generate_profile.py is missing.", + "hint": "Restore generate_profile.py script." + } + if not params_file.exists(): + return { + "name": "Generator Smoke", + "status": "FAIL", + "description": "templates/profile.params.yaml is missing.", + "hint": "Restore profile.params.yaml starter parameters." + } + + with tempfile.TemporaryDirectory(prefix="hermes-gen-") as tmpdir: + output_path = Path(tmpdir) / "generated" + + # Run generator + gen_proc = subprocess.run([ + sys.executable, + str(gen_script), + "--params", str(params_file), + "--output", str(output_path) + ], capture_output=True, text=True) + + if gen_proc.returncode != 0: + return { + "name": "Generator Smoke", + "status": "FAIL", + "description": "Profile generation failed.", + "hint": f"Error from generate_profile.py:
{gen_proc.stderr or gen_proc.stdout}" + } + + # Run validate_profile.py on generated output + val_script = output_path / "scripts" / "validate_profile.py" + if not val_script.exists(): + return { + "name": "Generator Smoke", + "status": "FAIL", + "description": "Generated profile does not contain validate_profile.py.", + "hint": "Ensure generate_profile.py includes validation script in target." + } + + val_proc = subprocess.run([ + sys.executable, + str(val_script), + str(output_path) + ], capture_output=True, text=True) + + if val_proc.returncode != 0: + return { + "name": "Generator Smoke", + "status": "FAIL", + "description": "Generated profile validation failed.", + "hint": f"Error validating generated profile:
{val_proc.stderr or val_proc.stdout}" + } + + return { + "name": "Generator Smoke", + "status": "PASS", + "description": "Generated profile compiled, output generated, and validated successfully.", + "hint": "" + } + + +def check_install_smoke(root: Path) -> dict[str, str]: + hermes_cli = shutil.which("hermes") + if not hermes_cli: + return { + "name": "Install Smoke", + "status": "SKIPPED", + "description": "Hermes CLI ('hermes') not found in PATH.", + "hint": "Install Hermes CLI or add it to PATH to enable this check." + } + + with tempfile.TemporaryDirectory(prefix="hermes-home-") as tmpdir: + env = os.environ.copy() + env["HERMES_HOME"] = tmpdir + + # Run hermes profile install + install_proc = subprocess.run([ + hermes_cli, "profile", "install", str(root), + "--name", "profile-smoke", "--yes" + ], env=env, capture_output=True, text=True) + + if install_proc.returncode != 0: + return { + "name": "Install Smoke", + "status": "FAIL", + "description": "Hermes profile install command failed.", + "hint": f"Output from hermes command:
{install_proc.stderr or install_proc.stdout}" + } + + # Verify key files are present in the installed destination + installed_profile = Path(tmpdir) / "profiles" / "profile-smoke" + required_installed_files = [ + "SOUL.md", + "distribution.yaml", + "scripts/generate_profile.py", + "templates/profile.params.yaml" + ] + + missing = [] + for rel_file in required_installed_files: + if not (installed_profile / rel_file).is_file(): + missing.append(rel_file) + + if missing: + return { + "name": "Install Smoke", + "status": "FAIL", + "description": f"Installed profile is missing files: {', '.join(missing)}.", + "hint": "Check distribution_owned section in distribution.yaml and ensure these files are listed." + } + + return { + "name": "Install Smoke", + "status": "PASS", + "description": "Profile successfully installed and key files validated.", + "hint": "" + } + + +def check_install_docs(root: Path) -> dict[str, str]: + repo_identifier = None + proc = run_git(["remote", "get-url", "origin"], root) + if proc.returncode == 0: + url = proc.stdout.strip() + match = re.search(r"(?:github\.com[:/])([^/]+/[^/.]+)(?:\.git)?", url) + if match: + repo_identifier = f"github.com/{match.group(1)}" + + if not repo_identifier: + meta_path = root / "github-repo-metadata.yaml" + if meta_path.exists(): + try: + meta = yaml.safe_load(meta_path.read_text(encoding="utf-8")) or {} + homepage = meta.get("homepage", "") + if homepage: + match = re.search(r"(?:github\.com[:/])([^/]+/[^/.]+)(?:\.git)?", homepage) + if match: + repo_identifier = f"github.com/{match.group(1)}" + except Exception: + pass + + # Gather markdown files to scan + md_files = [root / "README.md"] + docs_dir = root / "docs" + if docs_dir.exists(): + md_files.extend(docs_dir.rglob("*.md")) + + def search_docs(pattern: str) -> bool: + regex = re.compile(pattern, re.IGNORECASE) + for md_file in md_files: + if md_file.is_file(): + try: + content = md_file.read_text(encoding="utf-8") + if regex.search(content): + return True + except Exception: + pass + return False + + if repo_identifier: + pattern = rf"hermes\s+profile\s+install\s+(?:https?://)?{re.escape(repo_identifier)}" + if search_docs(pattern): + return { + "name": "Install Documentation", + "status": "PASS", + "description": f"Found installation command matching '{repo_identifier}' in documentation.", + "hint": "" + } + else: + return { + "name": "Install Documentation", + "status": "FAIL", + "description": f"Installation command for '{repo_identifier}' was not found in docs.", + "hint": f"Add the command `hermes profile install {repo_identifier}` to README.md or other documentation." + } + else: + pattern = r"hermes\s+profile\s+install" + if search_docs(pattern): + return { + "name": "Install Documentation", + "status": "PASS", + "description": "Found a generic installation command in documentation, but could not determine git remote.", + "hint": "" + } + else: + return { + "name": "Install Documentation", + "status": "FAIL", + "description": "No installation command 'hermes profile install' found in documentation.", + "hint": "Document the installation instructions using `hermes profile install ` in README.md." + } + + +def generate_markdown_report(results: list[dict[str, str]], base: str, current_version: str) -> str: + overall_status = "PASS" + for r in results: + if r["status"] == "FAIL": + overall_status = "FAIL" + break + + status_emoji = "🟢" if overall_status == "PASS" else "🔴" + + lines = [ + f"# Release Readiness Report", + f"", + f"- **Overall Status**: {status_emoji} **{overall_status}**", + f"- **Current Version**: `{current_version}`", + f"- **Base Ref**: `{base}`", + f"", + f"## Checklist", + f"", + f"| Check | Status | Description | Remediation Hint |", + f"| :--- | :--- | :--- | :--- |", + ] + + for r in results: + status_str = r["status"] + if status_str == "PASS": + status_cell = "🟢 PASS" + elif status_str == "FAIL": + status_cell = "🔴 FAIL" + elif status_str == "SKIPPED": + status_cell = "⚪ SKIPPED" + else: + status_cell = f"🟡 {status_str}" + + # Clean hint for table markdown formatting + hint_cell = r["hint"].replace("\n", "
") + lines.append(f"| {r['name']} | {status_cell} | {r['description']} | {hint_cell} |") + + lines.append("") + + if overall_status == "FAIL": + lines.append("### ⚠️ Action Required") + lines.append("Please resolve the failed checks highlighted above before publishing or tagging the release.") + lines.append("") + else: + lines.append("### 🎉 Ready for Release") + lines.append("All checks passed or skipped. This profile template distribution is ready to be tagged and released!") + lines.append("") + + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Check release readiness and generate a Markdown report") + parser.add_argument("--base", default="origin/main", help="Git base ref to compare against") + parser.add_argument("--root", default=".", help="Repository root") + parser.add_argument("--strict", action="store_true", help="Fail instead of skip when base ref is unavailable") + parser.add_argument("--output", help="Write Markdown report to this file") + args = parser.parse_args() + + root = Path(args.root).resolve() + + try: + current_version = read_current_version(root) + except Exception as exc: + print(f"ERROR: Cannot determine current version from distribution.yaml: {exc}") + return 1 + + results = [] + + # Run checks + results.append(check_version_discipline(root, args.base, args.strict)) + results.append(check_changelog_heading(root)) + results.append(check_profile_validation(root)) + results.append(check_runtime_files(root)) + results.append(check_secrets(root)) + results.append(check_generator_smoke(root)) + results.append(check_install_smoke(root)) + results.append(check_install_docs(root)) + + report = generate_markdown_report(results, args.base, current_version) + + print(report) + + if args.output: + try: + Path(args.output).write_text(report, encoding="utf-8") + print(f"Report successfully written to {args.output}") + except Exception as exc: + print(f"ERROR: Failed to write report to {args.output}: {exc}") + + # Determine exit code + for r in results: + if r["status"] == "FAIL": + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2a69876861b454daf5cba355beee192e6d850a31 Mon Sep 17 00:00:00 2001 From: therealsaitama Date: Fri, 26 Jun 2026 00:16:23 +0530 Subject: [PATCH 2/4] chore: bump version to 0.3.1 --- CHANGELOG.md | 4 ++++ distribution.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe64b1..8818ffb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this Hermes profile distribution are documented here. +## 0.3.1 + +- Added release readiness checklist workflow. + ## 0.3.0 - Clarified that this repository is a developer authoring system built on top of Hermes Agent's native profile distribution runtime. diff --git a/distribution.yaml b/distribution.yaml index af4b2ee..d4c5acf 100644 --- a/distribution.yaml +++ b/distribution.yaml @@ -1,5 +1,5 @@ name: hermes-profile-template -version: 0.3.0 +version: 0.3.1 description: "Starter distribution for building custom Hermes Agent profiles with AI assistance." hermes_requires: ">=0.12.0" author: "CodeGraphTheory" From d5dda18d58179e7de58ac7ad5e9d8a30fefae374 Mon Sep 17 00:00:00 2001 From: therealsaitama Date: Fri, 26 Jun 2026 00:16:44 +0530 Subject: [PATCH 3/4] refactor: replace old check_release_version.py with release_readiness.py --- .github/workflows/release-guard.yml | 2 +- docs/profile-distribution-contract.md | 2 +- scripts/check_release_version.py | 138 -------------------------- 3 files changed, 2 insertions(+), 140 deletions(-) delete mode 100644 scripts/check_release_version.py diff --git a/.github/workflows/release-guard.yml b/.github/workflows/release-guard.yml index 0781f9f..111768f 100644 --- a/.github/workflows/release-guard.yml +++ b/.github/workflows/release-guard.yml @@ -16,4 +16,4 @@ jobs: - name: Install dependencies run: python -m pip install -r requirements.txt - name: Check release metadata - run: python3 scripts/check_release_version.py --base origin/${{ github.base_ref }} --strict + run: python3 scripts/release_readiness.py --base origin/${{ github.base_ref }} --strict diff --git a/docs/profile-distribution-contract.md b/docs/profile-distribution-contract.md index 41d6fad..625fbea 100644 --- a/docs/profile-distribution-contract.md +++ b/docs/profile-distribution-contract.md @@ -20,7 +20,7 @@ This template provides developer-facing authoring and release tooling for creati - deterministic YAML-driven generation with `scripts/generate_profile.py` - publish-time validation with `scripts/validate_profile.py` - isolated install smoke tests with `scripts/smoke_install.sh` -- release metadata checks with `scripts/check_release_version.py` +- release readiness checklist with `scripts/release_readiness.py` - repeatable GitHub metadata setup with `scripts/apply_github_metadata.py` - CI workflows for validation and release hygiene - catalog snippets and explicit template lineage metadata diff --git a/scripts/check_release_version.py b/scripts/check_release_version.py deleted file mode 100644 index d030487..0000000 --- a/scripts/check_release_version.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python3 -"""Fail when release-relevant changes omit version or changelog updates.""" -from __future__ import annotations - -import argparse -import re -import subprocess -import sys -from pathlib import Path - -try: - import yaml -except ImportError as exc: # pragma: no cover - raise SystemExit("PyYAML is required. Install with: python3 -m pip install pyyaml") from exc - -RELEASE_RELEVANT_PREFIXES = ( - "SOUL.md", - "AGENTS.md", - "README.md", - "config.yaml", - "distribution.yaml", - ".env.EXAMPLE", - "mcp.json", - "skills/", - "templates/", - "scripts/", - ".github/workflows/", - "SECURITY.md", - "CONTRIBUTING.md", - "github-repo-metadata.yaml", - "requirements.txt", - "Makefile", - "docs/", -) - -IGNORED_PATHS = { - "CHANGELOG.md", -} - - -def run_git(args: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: - return subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True) - - -def read_current_version(root: Path) -> str: - data = yaml.safe_load((root / "distribution.yaml").read_text(encoding="utf-8")) or {} - version = str(data.get("version") or "").strip() - if not version: - raise ValueError("distribution.yaml missing version") - return version - - -def read_base_version(root: Path, base: str) -> str | None: - proc = run_git(["show", f"{base}:distribution.yaml"], root) - if proc.returncode != 0: - return None - data = yaml.safe_load(proc.stdout) or {} - return str(data.get("version") or "").strip() or None - - -def changed_files(root: Path, base: str) -> list[str] | None: - proc = run_git(["diff", "--name-only", f"{base}...HEAD"], root) - if proc.returncode != 0: - proc = run_git(["diff", "--name-only", base, "HEAD"], root) - if proc.returncode != 0: - return None - return [line.strip() for line in proc.stdout.splitlines() if line.strip()] - - -def is_release_relevant(path: str) -> bool: - if path in IGNORED_PATHS: - return False - return any(path == prefix.rstrip("/") or path.startswith(prefix) for prefix in RELEASE_RELEVANT_PREFIXES) - - -def changelog_has_version(root: Path, version: str) -> bool: - path = root / "CHANGELOG.md" - if not path.exists(): - return False - text = path.read_text(encoding="utf-8") - return re.search(rf"^##\s+\[?{re.escape(version)}\]?\b", text, flags=re.MULTILINE) is not None - - -def main() -> int: - parser = argparse.ArgumentParser(description="Check release version discipline") - parser.add_argument("--base", default="origin/main", help="Git base ref to compare against") - parser.add_argument("--root", default=".", help="Repository root") - parser.add_argument("--strict", action="store_true", help="Fail instead of skip when base ref is unavailable") - args = parser.parse_args() - - root = Path(args.root).resolve() - files = changed_files(root, args.base) - if files is None: - message = f"Could not compare against base ref {args.base}" - if args.strict: - print(f"ERROR: {message}") - return 1 - print(f"SKIP: {message}") - return 0 - - relevant = [path for path in files if is_release_relevant(path)] - if not relevant: - print("No release-relevant changes detected.") - return 0 - - current_version = read_current_version(root) - base_version = read_base_version(root, args.base) - if base_version is None: - message = f"Could not read distribution.yaml from {args.base}" - if args.strict: - print(f"ERROR: {message}") - return 1 - print(f"SKIP: {message}") - return 0 - - errors: list[str] = [] - if current_version == base_version: - errors.append( - f"distribution.yaml version did not change. Base and current version are both {current_version}." - ) - if not changelog_has_version(root, current_version): - errors.append(f"CHANGELOG.md is missing an entry for version {current_version}.") - - if errors: - print("Release version check failed.") - print("Release-relevant files changed:") - for path in relevant: - print(f"- {path}") - for error in errors: - print(f"ERROR: {error}") - return 1 - - print(f"Release version check passed for version {current_version}.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 31e33ff80299be032eeabb14cd2b82faba103b02 Mon Sep 17 00:00:00 2001 From: therealsaitama Date: Fri, 26 Jun 2026 00:44:51 +0530 Subject: [PATCH 4/4] feat: add profile quality scorecard checker with json and markdown output (#14) --- .github/workflows/validate.yml | 2 + Makefile | 9 +- README.md | 54 ++++ scripts/scorecard.py | 553 +++++++++++++++++++++++++++++++++ tests/test_scorecard.py | 157 ++++++++++ 5 files changed, 774 insertions(+), 1 deletion(-) create mode 100755 scripts/scorecard.py create mode 100644 tests/test_scorecard.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 199fcba..7799fe5 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -19,5 +19,7 @@ jobs: run: make compile - name: Validate repository profile distribution run: make validate + - name: Run profile quality scorecard + run: python3 scripts/scorecard.py - name: Smoke-test install path run: make smoke diff --git a/Makefile b/Makefile index 1fdda95..185fd8a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: deps validate compile generate-smoke smoke release-check clean +.PHONY: deps validate compile generate-smoke smoke scorecard test release-check clean PYTHON ?= python3 BASE ?= origin/main @@ -9,6 +9,7 @@ deps: validate: $(PYTHON) scripts/validate_profile.py . + $(PYTHON) -m unittest discover -s tests -p "test_*.py" compile: PYTHONDONTWRITEBYTECODE=1 $(PYTHON) -m py_compile scripts/*.py @@ -21,6 +22,12 @@ generate-smoke: smoke: scripts/smoke_install.sh +scorecard: + $(PYTHON) scripts/scorecard.py + +test: + $(PYTHON) -m unittest discover -s tests -p "test_*.py" + release-check: $(PYTHON) scripts/release_readiness.py --base $(BASE) diff --git a/README.md b/README.md index 7de196a..d50e809 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,60 @@ make smoke The smoke script validates the repository, compiles Python scripts without writing bytecode, generates and validates a profile from `templates/profile.params.yaml`, and installs into a temporary `HERMES_HOME` when the Hermes CLI is available. If you do not use `make`, run `python3 scripts/validate_profile.py .` and `scripts/smoke_install.sh` directly. +## Profile Quality Scorecard + +In addition to structural validation, this repository includes a profile quality scorecard that reports release readiness and search/organic discovery alignment. + +To generate a scorecard report for your repository: + +```bash +make scorecard +``` + +Or execute the script directly: + +```bash +python3 scripts/scorecard.py [path/to/profile] +``` + +### Checks & Deductions + +The scorecard starts at `100` points and evaluates the repository against 12 criteria grouped by severity: + +| Check | Severity | Description | Deduction | +| :--- | :--- | :--- | :--- | +| **Required Root Files Presence** | `FAIL` | Checks for mandatory files (`distribution.yaml`, `SOUL.md`, etc.). | `-15` | +| **Manifest Structure & Fields** | `FAIL` | Validates `distribution.yaml` format and fields. | `-15` | +| **Environment Variables in Example** | `FAIL` | Ensures `env_requires` variables exist in `.env.EXAMPLE`. | `-15` | +| **No Committed Runtime/Cache Files** | `FAIL` | Checks that no temporary, cache, or user database files are tracked. | `-15` | +| **No Exposed API Keys or Secrets** | `FAIL` | Checks for standard API keys or secret structures. | `-15` | +| **Skill Markdown Frontmatter** | `FAIL` | Checks that skill files under `skills/` contain valid frontmatter. | `-15` | +| **Python Script Syntax** | `FAIL` | Compiles Python scripts in-memory to catch syntax errors. | `-15` | +| **License File Presence** | `WARN` | Checks that a `LICENSE` file or metadata field exists. | `-5` | +| **Install Command in README** | `WARN` | Verifies installation instructions are documented. | `-5` | +| **GitHub Topic Metadata** | `WARN` | Verifies repository topic suggestions exist. | `-5` | +| **Changelog Version Consistency** | `WARN` | Verifies `CHANGELOG.md` matches the current version. | `-5` | +| **Smoke Testing Documentation** | `WARN` | Checks that smoke test instructions are in the README. | `-5` | + +### Output Formats + +- **Console/Terminal (Default)**: Visual output with colored status labels. +- **JSON (`--json`)**: Deterministic and structured output for integrations: + ```bash + python3 scripts/scorecard.py --json + ``` +- **Markdown (`--markdown`)**: A formatted table suitable for pull request comments or release summaries: + ```bash + python3 scripts/scorecard.py --markdown + ``` + +### Exit Codes + +- `0`: Scorecard passed with all checks OK, or with advisory warnings (`WARN`) only. +- `1`: Scorecard failed due to one or more hard/critical errors (`FAIL`). +- `2`: Command execution error (e.g., path does not exist). + + ## Release discipline diff --git a/scripts/scorecard.py b/scripts/scorecard.py new file mode 100755 index 0000000..6390d63 --- /dev/null +++ b/scripts/scorecard.py @@ -0,0 +1,553 @@ +#!/usr/bin/env python3 +"""Hermes Profile Quality Scorecard. + +Evaluates a Hermes profile repository for release and organic discovery readiness, +producing human-readable, JSON, and Markdown outputs. + +Score calculation: +- Starts at 100. +- Deducts 15 points for each failed check of severity 'FAIL' (hard failure). +- Deducts 5 points for each failed check of severity 'WARN' (advisory warning). +- Minimum score is 0. + +Exit codes: +- 0: All checks passed, or only advisory warnings (WARN) failed. +- 1: One or more critical checks (FAIL) failed. +- 2: CLI error (e.g. invalid path). +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + yaml = None + +SECRET_PATTERNS = [ + re.compile(r"ghp_[A-Za-z0-9_]{20,}"), + re.compile(r"gho_[A-Za-z0-9_]{20,}"), + re.compile(r"sk-[A-Za-z0-9]{20,}"), + re.compile(r"xox[baprs]-[A-Za-z0-9-]{20,}"), + re.compile(r"AKIA[0-9A-Z]{16}"), +] + +USER_OWNED = { + ".env", + "auth.json", + "state.db", + "state.db-shm", + "state.db-wal", + "memories", + "sessions", + "logs", + "workspace", + "plans", + "local", + "cache", +} + +FORBIDDEN_DIR_NAMES = USER_OWNED | { + "__pycache__", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + "htmlcov", + "dist", + "build", + "hook-sessions", +} + +FORBIDDEN_FILE_NAMES = { + ".coverage", + "coverage.xml", +} + +FORBIDDEN_SUFFIXES = ( + ".pyc", + ".pyo", + ".pyd", +) + +REQUIRED_ROOT = ["distribution.yaml", "SOUL.md", "README.md", "AGENTS.md", "config.yaml", ".env.EXAMPLE"] + + +def get_git_files(root: Path) -> list[Path] | None: + """Get tracked and unignored files using git ls-files if it is a git repository.""" + if (root / ".git").exists(): + import subprocess + proc = subprocess.run( + ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], + cwd=root, + text=False, + capture_output=True, + ) + if proc.returncode == 0: + rels = [item.decode("utf-8") for item in proc.stdout.split(b"\0") if item] + return [root / rel for rel in rels] + return None + + +def get_all_files(root: Path) -> list[Path]: + """Get all files recursively from path, excluding hidden dot directories like .git.""" + files = [] + for p in root.rglob("*"): + if p.is_file(): + # Exclude files inside dot folders (like .git, .pytest_cache) + if any(part.startswith(".") and part != "." for part in p.relative_to(root).parts[:-1]): + continue + files.append(p) + return files + + +def check_required_files(root: Path) -> tuple[str, str]: + missing = [f for f in REQUIRED_ROOT if not (root / f).is_file()] + if missing: + return "FAIL", f"Missing required root files: {', '.join(missing)}" + return "PASS", "All required root files are present." + + +def check_manifest_fields(root: Path) -> tuple[str, str]: + manifest_path = root / "distribution.yaml" + if not manifest_path.is_file(): + return "FAIL", "distribution.yaml is missing." + if yaml is None: + return "FAIL", "PyYAML is required but not installed." + try: + data = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {} + except Exception as exc: + return "FAIL", f"Invalid YAML in distribution.yaml: {exc}" + if not isinstance(data, dict): + return "FAIL", "distribution.yaml must be a YAML mapping." + + missing_fields = [f for f in ["name", "version", "description"] if not str(data.get(f, "")).strip()] + if missing_fields: + return "FAIL", f"distribution.yaml missing required field(s): {', '.join(missing_fields)}" + + name = str(data.get("name", "")) + if name and not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", name): + return "FAIL", f"distribution.yaml name '{name}' must be lowercase kebab-case." + + return "PASS", f"Manifest has valid required fields (name: {name}, version: {data.get('version')})." + + +def check_env_vars(root: Path) -> tuple[str, str]: + manifest_path = root / "distribution.yaml" + if not manifest_path.is_file(): + return "FAIL", "distribution.yaml is missing." + if yaml is None: + return "FAIL", "PyYAML is required but not installed." + try: + data = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {} + except Exception: + return "FAIL", "Invalid distribution.yaml." + env_requires = data.get("env_requires", []) + if not env_requires: + return "PASS", "No environment variables are required by this profile." + if not isinstance(env_requires, list): + return "FAIL", "distribution.yaml env_requires must be a list." + + env_example_path = root / ".env.EXAMPLE" + if not env_example_path.is_file(): + return "FAIL", ".env.EXAMPLE file is missing." + + try: + example_content = env_example_path.read_text(encoding="utf-8") + except Exception as exc: + return "FAIL", f"Could not read .env.EXAMPLE: {exc}" + + missing_vars = [] + for item in env_requires: + if isinstance(item, dict) and item.get("name"): + name = item["name"] + if name not in example_content: + missing_vars.append(name) + elif isinstance(item, str): + if item not in example_content: + missing_vars.append(item) + + if missing_vars: + return "FAIL", f"Required env vars are missing from .env.EXAMPLE: {', '.join(missing_vars)}" + + return "PASS", "All required environment variables are documented in .env.EXAMPLE." + + +def check_no_runtime_files(root: Path) -> tuple[str, str]: + forbidden_dirs = FORBIDDEN_DIR_NAMES + forbidden_files = FORBIDDEN_FILE_NAMES | USER_OWNED + + committed_forbidden = [] + seen_dirs = set() + + paths = get_git_files(root) + if paths is None: + paths = get_all_files(root) + + for path in paths: + if ".git" in path.parts: + continue + rel = path.relative_to(root) + for parent in rel.parents: + if str(parent) == "." or parent in seen_dirs: + continue + seen_dirs.add(parent) + if parent.name in forbidden_dirs: + committed_forbidden.append(f"directory '{parent}'") + if path.name in forbidden_files: + committed_forbidden.append(f"file '{rel}'") + if path.name.endswith(FORBIDDEN_SUFFIXES): + committed_forbidden.append(f"compiled artifact '{rel}'") + + if committed_forbidden: + return "FAIL", f"Committed forbidden runtime or cache paths found: {', '.join(committed_forbidden)}" + return "PASS", "No runtime, cache, or user-owned files are tracked in the repository." + + +def check_no_secrets(root: Path) -> tuple[str, str]: + skip_dirs = {".git", "node_modules", ".venv", "venv", "__pycache__"} + paths = get_git_files(root) + if paths is None: + paths = get_all_files(root) + + exposed = [] + for path in paths: + if not path.is_file() or any(part in skip_dirs for part in path.parts): + continue + if path.suffix.lower() in {".png", ".jpg", ".jpeg", ".gif", ".pdf"}: + continue + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + for pattern in SECRET_PATTERNS: + if pattern.search(text): + exposed.append(str(path.relative_to(root))) + break + if exposed: + return "FAIL", f"Exposed secret patterns found in files: {', '.join(exposed)}" + return "PASS", "No exposed credentials or secret keys detected." + + +def check_skill_frontmatter(root: Path) -> tuple[str, str]: + skills_dir = root / "skills" + if not skills_dir.exists(): + return "PASS", "No skills directory present." + + skill_files = list(skills_dir.rglob("SKILL.md")) + if not skill_files: + return "PASS", "No SKILL.md files found." + + failures = [] + warnings = [] + + for skill_md in skill_files: + rel = skill_md.relative_to(root) + try: + text = skill_md.read_text(encoding="utf-8") + except Exception as exc: + failures.append(f"Could not read {rel}: {exc}") + continue + + if not text.startswith("---\n"): + failures.append(f"{rel} missing frontmatter marker '---'") + continue + + parts = text.split("---", 2) + if len(parts) < 3: + failures.append(f"{rel} frontmatter not closed") + continue + + if yaml is None: + failures.append(f"PyYAML is required to parse frontmatter of {rel}") + continue + + try: + meta = yaml.safe_load(parts[1]) or {} + except Exception as exc: + failures.append(f"Invalid YAML in frontmatter of {rel}: {exc}") + continue + + if not isinstance(meta, dict): + failures.append(f"Frontmatter in {rel} must be a YAML mapping") + continue + + for key in ["name", "description"]: + if not meta.get(key): + warnings.append(f"Skill {rel} missing frontmatter field: {key}") + + if failures: + return "FAIL", f"Critical frontmatter errors: {'; '.join(failures)}" + if warnings: + return "WARN", f"Advisory frontmatter warnings: {'; '.join(warnings)}" + return "PASS", "All skills contain valid name and description metadata in frontmatter." + + +def check_script_compilation(root: Path) -> tuple[str, str]: + scripts_dir = root / "scripts" + if not scripts_dir.is_dir(): + return "PASS", "No scripts directory to compile." + python_files = [p for p in scripts_dir.glob("*.py")] + if not python_files: + return "PASS", "No Python scripts found to compile." + errors = [] + for f in python_files: + try: + content = f.read_text(encoding="utf-8") + compile(content, str(f), "exec") + except Exception as exc: + errors.append(f"{f.name}: {exc}") + if errors: + return "FAIL", f"Python script compilation errors: {', '.join(errors)}" + return "PASS", "All python scripts compile successfully." + + +def check_license_presence(root: Path) -> tuple[str, str]: + license_files = [] + if root.exists(): + license_files = [f for f in os.listdir(root) if f.upper() in ["LICENSE", "LICENSE.TXT", "LICENSE.MD"] if (root / f).is_file()] + if license_files: + return "PASS", f"License file '{license_files[0]}' found." + + manifest_path = root / "distribution.yaml" + if manifest_path.is_file() and yaml is not None: + try: + data = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {} + if str(data.get("license", "")).strip(): + return "PASS", f"License '{data['license']}' specified in distribution.yaml." + except Exception: + pass + + return "WARN", "No LICENSE file found in repository root." + + +def check_install_command(root: Path) -> tuple[str, str]: + readme_path = root / "README.md" + if not readme_path.is_file(): + return "WARN", "README.md is missing." + try: + content = readme_path.read_text(encoding="utf-8") + except Exception as exc: + return "WARN", f"Could not read README.md: {exc}" + + if re.search(r"hermes\s+profile\s+install", content): + return "PASS", "Installation command documented in README.md." + return "WARN", "No 'hermes profile install' command found in README.md." + + +def check_github_topics(root: Path) -> tuple[str, str]: + meta_path = root / "github-repo-metadata.yaml" + if not meta_path.is_file(): + return "WARN", "github-repo-metadata.yaml is missing." + if yaml is None: + return "WARN", "PyYAML missing, cannot check github-repo-metadata.yaml." + try: + data = yaml.safe_load(meta_path.read_text(encoding="utf-8")) or {} + except Exception as exc: + return "WARN", f"Invalid YAML in github-repo-metadata.yaml: {exc}" + + if isinstance(data, dict) and data.get("topics") and isinstance(data["topics"], list) and len(data["topics"]) > 0: + return "PASS", f"GitHub topic recommendations found: {', '.join(data['topics'])}." + + return "WARN", "No topic recommendations found in github-repo-metadata.yaml." + + +def check_changelog_consistency(root: Path) -> tuple[str, str]: + manifest_path = root / "distribution.yaml" + if not manifest_path.is_file(): + return "WARN", "distribution.yaml missing." + if yaml is None: + return "WARN", "PyYAML missing, cannot check version." + try: + data = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {} + version = str(data.get("version", "")).strip() + except Exception: + return "WARN", "Could not parse version from distribution.yaml." + + changelog_path = root / "CHANGELOG.md" + if not changelog_path.is_file(): + return "WARN", "CHANGELOG.md is missing." + + try: + content = changelog_path.read_text(encoding="utf-8") + except Exception as exc: + return "WARN", f"Could not read CHANGELOG.md: {exc}" + + if not version: + return "WARN", "Version in distribution.yaml is empty." + + escaped_version = re.escape(version) + pattern = rf"(?m)^#+\s+(?:v|\[)?{escaped_version}(?:\])?\b" + if re.search(pattern, content, re.IGNORECASE): + return "PASS", f"Changelog contains matching entry for version '{version}'." + + return "WARN", f"Changelog is missing an entry for the current version '{version}'." + + +def check_smoke_command(root: Path) -> tuple[str, str]: + readme_path = root / "README.md" + if not readme_path.is_file(): + return "WARN", "README.md is missing." + try: + content = readme_path.read_text(encoding="utf-8") + except Exception as exc: + return "WARN", f"Could not read README.md: {exc}" + + if "make smoke" in content or "smoke_install.sh" in content: + return "PASS", "Smoke test commands/instructions found in README.md." + + return "WARN", "README.md does not document how to run smoke tests ('make smoke' or 'smoke_install.sh')." + + +def run_scorecard(root: Path) -> dict: + checks_config = [ + ("required-files", "Required Root Files Presence", "FAIL", check_required_files), + ("manifest-fields", "Manifest Structure & Mandatory Fields", "FAIL", check_manifest_fields), + ("env-vars", "Environment Variables in .env.EXAMPLE", "FAIL", check_env_vars), + ("no-runtime-files", "No Committed Runtime or Cache Files", "FAIL", check_no_runtime_files), + ("no-secrets", "No Exposed API Keys or Secrets", "FAIL", check_no_secrets), + ("skill-frontmatter", "Skill Markdown Frontmatter Validity", "FAIL", check_skill_frontmatter), + ("script-compilation", "Python Script Syntax Verification", "FAIL", check_script_compilation), + ("license-presence", "License File Presence", "WARN", check_license_presence), + ("install-command", "Install Command in Documentation", "WARN", check_install_command), + ("github-topics", "GitHub Topic Metadata Recommendations", "WARN", check_github_topics), + ("changelog-consistency", "Changelog Version Consistency", "WARN", check_changelog_consistency), + ("smoke-command", "Smoke Testing Commands in Documentation", "WARN", check_smoke_command), + ] + + score = 100 + failed_count = 0 + warning_count = 0 + passed_count = 0 + checks_results = [] + + for cid, cname, severity, func in checks_config: + try: + status, message = func(root) + except Exception as exc: + status, message = "FAIL", f"Internal check runner error: {exc}" + + if status == "FAIL": + failed_count += 1 + if severity == "FAIL": + score -= 15 + else: + score -= 5 + elif status == "WARN": + warning_count += 1 + # A WARN status means it failed an advisory check + score -= 5 + else: + passed_count += 1 + + checks_results.append({ + "id": cid, + "name": cname, + "severity": severity, + "status": status, + "message": message + }) + + score = max(0, score) + status = "FAIL" if any(r["status"] == "FAIL" and r["severity"] == "FAIL" for r in checks_results) else "PASS" + + return { + "scorecard_version": "1.0", + "status": status, + "score": score, + "passed_count": passed_count, + "warning_count": warning_count, + "failed_count": failed_count, + "checks": checks_results + } + + +def print_console(res: dict) -> None: + # Use color escapes only if outputting to a TTY + use_color = sys.stdout.isatty() + + GREEN = "\033[92m" if use_color else "" + YELLOW = "\033[93m" if use_color else "" + RED = "\033[91m" if use_color else "" + RESET = "\033[0m" if use_color else "" + BOLD = "\033[1m" if use_color else "" + + status_color = GREEN if res["status"] == "PASS" else RED + + print("=" * 60) + print(f"{BOLD}HERMES PROFILE QUALITY SCORECARD{RESET}") + print("=" * 60) + print(f"Overall Status : {status_color}{res['status']}{RESET}") + print(f"Quality Score : {BOLD}{res['score']}/100{RESET}") + print(f"Summary : {GREEN}{res['passed_count']} Passed{RESET}, {YELLOW}{res['warning_count']} Warnings{RESET}, {RED}{res['failed_count']} Failed{RESET}") + print("-" * 60) + + for r in res["checks"]: + if r["status"] == "PASS": + status_str = f"{GREEN}PASS{RESET}" + elif r["status"] == "WARN": + status_str = f"{YELLOW}WARN{RESET}" + else: + status_str = f"{RED}FAIL{RESET}" + + # Highlight checks that caused hard failure vs advisory warnings + severity_label = f" ({r['severity']})" if r["status"] != "PASS" else "" + print(f"[{status_str}]{severity_label} {r['name']}") + print(f" {r['message']}") + print("=" * 60) + + +def print_markdown(res: dict) -> None: + status_emoji = "🟢 PASS" if res["status"] == "PASS" else "🔴 FAIL" + print("# Hermes Profile Quality Scorecard") + print() + print(f"- **Overall Status**: {status_emoji}") + print(f"- **Quality Score**: `{res['score']}/100`") + print(f"- **Summary**: 🟢 `{res['passed_count']}` Passed | 🟡 `{res['warning_count']}` Warnings | 🔴 `{res['failed_count']}` Failed") + print() + print("## Checklist Details") + print() + print("| Status | Severity | Check | Details / Remediation |") + print("| :--- | :--- | :--- | :--- |") + for r in res["checks"]: + if r["status"] == "PASS": + status_cell = "🟢 PASS" + elif r["status"] == "WARN": + status_cell = "🟡 WARN" + else: + status_cell = "🔴 FAIL" + print(f"| {status_cell} | `{r['severity']}` | {r['name']} | {r['message']} |") + print() + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate a quality scorecard for a Hermes profile") + parser.add_argument("path", nargs="?", default=".", help="Path to the repository root") + parser.add_argument("--json", action="store_true", help="Output results in JSON format") + parser.add_argument("--markdown", action="store_true", help="Output results in Markdown format") + args = parser.parse_args() + + root = Path(args.path).resolve() + if not root.exists(): + print(f"ERROR: path does not exist: {root}", file=sys.stderr) + return 2 + + res = run_scorecard(root) + + if args.json: + print(json.dumps(res, indent=2)) + elif args.markdown: + print_markdown(res) + else: + print_console(res) + + # Return 1 if there is a hard failure (overall status FAIL) + return 0 if res["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_scorecard.py b/tests/test_scorecard.py new file mode 100644 index 0000000..868702f --- /dev/null +++ b/tests/test_scorecard.py @@ -0,0 +1,157 @@ +import json +import os +import sys +import tempfile +import unittest +from io import StringIO +from pathlib import Path +from unittest.mock import patch + +# Add scripts directory to path to allow importing scorecard +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import scorecard + + +class TestProfileScorecard(unittest.TestCase): + def setUp(self): + # Create a temporary directory for the test workspace + self.test_dir = tempfile.TemporaryDirectory() + self.root = Path(self.test_dir.name) + + def tearDown(self): + # Clean up temporary directory + self.test_dir.cleanup() + + def create_minimal_valid_structure(self): + """Helper to create a minimal passing structure.""" + # Required files + (self.root / "SOUL.md").write_text("Mission description", encoding="utf-8") + (self.root / "README.md").write_text( + "Install: `hermes profile install` \nSmoke test: `make smoke`", encoding="utf-8" + ) + (self.root / "AGENTS.md").write_text("Agent rules", encoding="utf-8") + (self.root / "config.yaml").write_text("model: default", encoding="utf-8") + (self.root / ".env.EXAMPLE").write_text("API_KEY=\nDATABASE_URL=", encoding="utf-8") + + # Manifest + (self.root / "distribution.yaml").write_text( + "name: test-profile\nversion: 0.1.0\ndescription: A test profile\n", encoding="utf-8" + ) + + # License + (self.root / "LICENSE").write_text("MIT License", encoding="utf-8") + + # GitHub topics + (self.root / "github-repo-metadata.yaml").write_text("topics:\n - hermes-agent\n", encoding="utf-8") + + # Changelog + (self.root / "CHANGELOG.md").write_text("## 0.1.0\n- Initial release", encoding="utf-8") + + # Create clean scripts dir + os.makedirs(self.root / "scripts", exist_ok=True) + (self.root / "scripts" / "helper.py").write_text("print('hello')", encoding="utf-8") + + def test_perfect_scorecard_passes(self): + """Test a pristine profile distribution gets a score of 100 and overall PASS.""" + self.create_minimal_valid_structure() + res = scorecard.run_scorecard(self.root) + + self.assertEqual(res["status"], "PASS") + self.assertEqual(res["score"], 100) + self.assertEqual(res["passed_count"], 12) + self.assertEqual(res["failed_count"], 0) + self.assertEqual(res["warning_count"], 0) + + def test_advisory_warning_only(self): + """Test that advisory warnings (e.g. missing LICENSE, missing docs) deduct points but PASS.""" + self.create_minimal_valid_structure() + # Remove LICENSE file and github topics file to trigger warnings + os.remove(self.root / "LICENSE") + os.remove(self.root / "github-repo-metadata.yaml") + + res = scorecard.run_scorecard(self.root) + + # Scorecard should still pass but with a reduced score + self.assertEqual(res["status"], "PASS") + self.assertTrue(res["score"] < 100) + self.assertEqual(res["warning_count"], 2) + self.assertEqual(res["failed_count"], 0) + + def test_critical_failure_exit_code(self): + """Test that a critical failure (e.g. missing SOUL.md) yields FAIL status.""" + self.create_minimal_valid_structure() + # Remove required file SOUL.md + os.remove(self.root / "SOUL.md") + + res = scorecard.run_scorecard(self.root) + + self.assertEqual(res["status"], "FAIL") + self.assertEqual(res["failed_count"], 1) + self.assertTrue(res["score"] < 100) + + def test_malformed_manifest(self): + """Test that a malformed manifest (invalid YAML) yields FAIL status.""" + self.create_minimal_valid_structure() + # Write invalid YAML to manifest + (self.root / "distribution.yaml").write_text("name: : : : invalid", encoding="utf-8") + + res = scorecard.run_scorecard(self.root) + + self.assertEqual(res["status"], "FAIL") + # Manifest field failure should occur + manifest_check = next(c for c in res["checks"] if c["id"] == "manifest-fields") + self.assertEqual(manifest_check["status"], "FAIL") + + def test_skill_frontmatter_errors(self): + """Test skill markdown frontmatter parsing validation.""" + self.create_minimal_valid_structure() + skills_dir = self.root / "skills" / "test_skill" + os.makedirs(skills_dir, exist_ok=True) + + # Write malformed skill frontmatter (missing closing marker) + (skills_dir / "SKILL.md").write_text("---\nname: Test Skill\nDescription: Missing closing", encoding="utf-8") + + res = scorecard.run_scorecard(self.root) + self.assertEqual(res["status"], "FAIL") + + # Now fix frontmatter format but omit name/description + (skills_dir / "SKILL.md").write_text("---\nauthor: Saitama\n---\nContent", encoding="utf-8") + res = scorecard.run_scorecard(self.root) + # Missing fields should only warn (advisory), so overall status remains PASS + self.assertEqual(res["status"], "PASS") + skill_check = next(c for c in res["checks"] if c["id"] == "skill-frontmatter") + self.assertEqual(skill_check["status"], "WARN") + + def test_json_and_markdown_output(self): + """Test json and markdown output flags in CLI mode.""" + self.create_minimal_valid_structure() + + # Test JSON CLI output + with patch("sys.argv", ["scorecard.py", str(self.root), "--json"]): + out = StringIO() + with patch("sys.stdout", out): + try: + scorecard.main() + except SystemExit as exc: + self.assertEqual(exc.code, 0) + + data = json.loads(out.getvalue()) + self.assertEqual(data["status"], "PASS") + self.assertEqual(data["score"], 100) + + # Test Markdown CLI output + with patch("sys.argv", ["scorecard.py", str(self.root), "--markdown"]): + out = StringIO() + with patch("sys.stdout", out): + try: + scorecard.main() + except SystemExit as exc: + self.assertEqual(exc.code, 0) + + md_content = out.getvalue() + self.assertIn("# Hermes Profile Quality Scorecard", md_content) + self.assertIn("## Checklist Details", md_content) + + +if __name__ == "__main__": + unittest.main()