feat(eval): cross-harness eval/benchmark framework (cli-anything-eval)

.github/scripts/generate_meta_skill.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Generate cli-hub-skill/SKILL.md from registry.json and public_registry.json."""
+"""Generate cli-hub-skill/SKILL.md from registry.json, public_registry.json, and matrix_registry.json."""
 import json
 from pathlib import Path
 from collections import defaultdict
@@ -8,6 +8,7 @@ def main():
     repo_root = Path(__file__).parent.parent.parent
     registry_path = repo_root / 'registry.json'
     public_registry_path = repo_root / 'public_registry.json'
+    matrix_registry_path = repo_root / 'matrix_registry.json'
     output_path = repo_root / 'cli-hub-skill' / 'SKILL.md'
 
     with open(registry_path) as f:
@@ -19,6 +20,12 @@ def main():
             public_data = json.load(f)
         public_clis = public_data.get('clis', [])
 
+    matrices = []
+    if matrix_registry_path.exists():
+        with open(matrix_registry_path) as f:
+            matrix_data = json.load(f)
+        matrices = matrix_data.get('matrices', [])
+
     total_count = len(data['clis']) + len(public_clis)
 
     # Group harness CLIs by category
@@ -65,6 +72,24 @@ def main():
         "cli-hub launch <name> [args...]",
         "```",
         "",
+        "## CLI Matrices",
+        "",
+        f"`cli-hub` also ships {len(matrices)} curated cross-tool matrices: install one name to pull in a whole workflow kit and read its dedicated SKILL.md.",
+        "",
+        "```bash",
+        "# Browse curated matrices",
+        "cli-hub matrix list",
+        "",
+        "# Inspect one matrix",
+        "cli-hub matrix info video-creation",
+        "",
+        "# Check which providers are available locally",
+        "cli-hub matrix preflight video-creation --json",
+        "",
+        "# Install the whole matrix",
+        "cli-hub matrix install video-creation",
+        "```",
+        "",
         "## CLI-Anything Harness CLIs",
         "",
         f"Stateful, agent-native wrappers for {len(data['clis'])} GUI applications. All support `--json` output, REPL mode, and undo/redo.",
@@ -97,15 +122,36 @@ def main():
         clis = public_by_category[category]
         lines.append(f"### {category.title()}")
         lines.append("")
-        lines.append("| Name | Description | Entry Point | Install |")
-        lines.append("|------|-------------|-------------|---------|")
+        lines.append("| Name | Description | Entry Point | Install | Skill |")
+        lines.append("|------|-------------|-------------|---------|-------|")
 
         for cli in sorted(clis, key=lambda x: x['name']):
             name = cli['display_name']
             desc = cli['description']
             entry = f"`{cli['entry_point']}`"
             install = f"`cli-hub install {cli['name']}`"
-            lines.append(f"| **{name}** | {desc} | {entry} | {install} |")
+            skill = cli.get('skill_md') or '—'
+            skill_cell = f"`{skill}`" if not str(skill).startswith("http") else skill
+            lines.append(f"| **{name}** | {desc} | {entry} | {install} | {skill_cell} |")
+
+        lines.append("")
+
+    if matrices:
+        lines.extend([
+            "## Curated Matrices",
+            "",
+            "Each matrix is a curated multi-CLI workflow pulled from the CLI Matrix. Installing a matrix installs all member CLIs and points you at a matrix-specific SKILL.md.",
+            "",
+            "| Matrix | Description | CLIs | Install | Skill |",
+            "|--------|-------------|------|---------|-------|",
+        ])
+
+        for matrix in sorted(matrices, key=lambda x: x['name']):
+            skill = matrix.get('skill_md') or '—'
+            install = f"`cli-hub matrix install {matrix['name']}`"
+            lines.append(
+                f"| **{matrix['display_name']}** | {matrix['description']} | {len(matrix.get('clis', []))} | {install} | `{skill}` |"
+            )
 
         lines.append("")
 
@@ -119,6 +165,7 @@ def main():
         "- **uv CLIs**: installed via `uv tool install`",
         "- **brew/script CLIs**: installed via the tool's native installer",
         "- **bundled CLIs**: detected from PATH (pre-installed with the host app)",
+        "- **Matrices**: install a curated set of harness and public CLIs in one command",
         "",
         "## Harness CLI Usage Pattern",
         "",
@@ -152,7 +199,10 @@ def main():
 
     output_path.parent.mkdir(parents=True, exist_ok=True)
     output_path.write_text('\n'.join(lines) + '\n')
-    print(f"Generated meta-skill with {len(data['clis'])} harness CLIs + {len(public_clis)} public CLIs ({total_count} total) at {output_path}")
+    print(
+        f"Generated meta-skill with {len(data['clis'])} harness CLIs + "
+        f"{len(public_clis)} public CLIs + {len(matrices)} matrices at {output_path}"
+    )
 
 if __name__ == '__main__':
     main()

.github/workflows/deploy-pages.yml

@@ -8,7 +8,9 @@ on:
       - '*/agent-harness/**'
       - 'registry.json'
       - 'public_registry.json'
+      - 'matrix_registry.json'
       - 'cli-hub/**'
+      - 'cli-hub-matrix/**'
       - 'docs/hub/**'
       - '.github/workflows/deploy-pages.yml'
       - '.github/scripts/update_registry_dates.py'
@@ -63,12 +65,25 @@ jobs:
           cp registry.json docs/hub/registry.json
           cp public_registry.json docs/hub/public_registry.json
 
+      - name: Copy matrix_registry.json to hub for cli-hub access
+        run: cp matrix_registry.json docs/hub/matrix_registry.json
+
       - name: Build with Jekyll
         uses: actions/jekyll-build-pages@v1
         with:
           source: ./docs/hub
           destination: ./docs/_site
 
+      # Copied after the Jekyll build so SKILL.md front matter is served
+      # verbatim instead of being converted to HTML. Published at
+      # https://hkuds.github.io/CLI-Anything/matrix/<name>/{SKILL.md,references/,scripts/}
+      # (consumed by cli_hub.matrix_skill's published-URL fallback).
+      - name: Copy matrix skill content into site
+        run: |
+          sudo mkdir -p docs/_site/matrix
+          sudo rsync -a --exclude='__pycache__' --exclude='*.pyc' --exclude='*.pyo' \
+            cli-hub-matrix/ docs/_site/matrix/
+
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:

.gitignore

@@ -39,6 +39,7 @@
 !/openclaw-skill/
 !/cli-hub-meta-skill/
 !/cli-hub/
+!/cli-anything-eval/
 !/cli-hub-matrix/
 !/skills/
 !/skills/**
@@ -326,18 +327,14 @@ assets/gen_typing_gif.py
 !/registry.json
 !/public_registry.json
 !/matrix_registry.json
-!/docs/
-/docs/*
-!/docs/PREVIEW_PROTOCOL.md
-!/docs/PREVIEW_PROGRESS.md
-!/docs/FREECAD_VIDEO_REFERENCE.md
-!/docs/PREVIEW_MECHANISM_PROGRESS.md
-!/docs/scripts/
-!/docs/scripts/**
-/docs/scripts/__pycache__/
-/docs/scripts/**/*.pyc
-!/docs/hub/
-/docs/hub/registry-dates.json
+# docs/* is always ignored — working documents stay local and are never committed.
+# (Previously-tracked files under docs/ remain in the index until explicitly removed.)
+/docs/
+# CLI-Matrix is unshipped/confidential — never track anything under docs/cli-matrix/.
+/docs/cli-matrix/
+
+# Build-time vendored matrix skill content (generated by cli-hub setup.py build_py/sdist)
+/cli-hub/cli_hub/_matrix_data/
 !/notebooklm/
 /notebooklm/*
 /notebooklm/.*

README.md

@@ -1672,7 +1672,7 @@ We welcome contributions! CLI-Anything is designed to be extensible:
 ### Roadmap
 
 - [ ] Support for more application categories (CAD, DAW, IDE, EDA, scientific tools)
-- [ ] Benchmark suite for agent task completion rates
+- [x] Benchmark suite for agent task completion rates
 - [ ] Community-contributed CLI harnesses for internal/custom software
 - [ ] Integration with additional agent frameworks beyond Claude Code
 - [ ] Support packaging APIs for closed-source software and web services into CLIs

audacity/agent-harness/cli_anything/audacity/audacity_cli.py

@@ -40,6 +40,7 @@
 _session: Optional[Session] = None
 _json_output = False
 _repl_mode = False
+_dry_run = False
 
 
 def get_session() -> Session:
@@ -63,6 +64,15 @@ def output(data, message: str = ""):
             click.echo(str(data))
 
 
+def autosave_session_if_needed() -> None:
+    """Persist one-shot mutations immediately when working from a project file."""
+    if _repl_mode or _dry_run:
+        return
+    sess = get_session()
+    if sess.has_project() and sess._modified and sess.project_path:
+        sess.save_session()
+
+
 def _print_dict(d: dict, indent: int = 0):
     prefix = "  " * indent
     for k, v in d.items():
@@ -129,8 +139,9 @@ def cli(ctx, use_json, project_path, dry_run):
 
     Run without a subcommand to enter interactive REPL mode.
     """
-    global _json_output
+    global _json_output, _dry_run
     _json_output = use_json
+    _dry_run = dry_run
 
     if project_path:
         sess = get_session()
@@ -229,6 +240,7 @@ def project_settings(sample_rate, bit_depth, channels):
         sess.snapshot("Change settings")
         result = proj_mod.set_settings(proj, sample_rate, bit_depth, channels)
         output(result, "Settings updated:")
+        autosave_session_if_needed()
     else:
         output(proj.get("settings", {}), "Project settings:")
 
@@ -264,6 +276,7 @@ def track_add(name, track_type, volume, pan):
         volume=volume, pan=pan,
     )
     output(result, f"Added track: {result['name']}")
+    autosave_session_if_needed()
 
 
 @track.command("remove")
@@ -275,6 +288,7 @@ def track_remove(index):
     sess.snapshot(f"Remove track {index}")
     removed = track_mod.remove_track(sess.get_project(), index)
     output(removed, f"Removed track: {removed.get('name', '')}")
+    autosave_session_if_needed()
 
 
 @track.command("list")
@@ -298,6 +312,7 @@ def track_set(index, prop, value):
     result = track_mod.set_track_property(sess.get_project(), index, prop, value)
     output({"track": index, "property": prop, "value": value},
            f"Set track {index} {prop} = {value}")
+    autosave_session_if_needed()
 
 
 # -- Clip Commands ---------------------------------------------------------
@@ -336,6 +351,7 @@ def clip_add(track_index, source, name, start, end, trim_start, trim_end, volume
         trim_start=trim_start, trim_end=trim_end, volume=volume,
     )
     output(result, f"Added clip: {result['name']}")
+    autosave_session_if_needed()
 
 
 @clip.command("remove")
@@ -348,6 +364,7 @@ def clip_remove(track_index, clip_index):
     sess.snapshot(f"Remove clip {clip_index} from track {track_index}")
     removed = clip_mod.remove_clip(sess.get_project(), track_index, clip_index)
     output(removed, f"Removed clip: {removed.get('name', '')}")
+    autosave_session_if_needed()
 
 
 @clip.command("trim")
@@ -365,6 +382,7 @@ def clip_trim(track_index, clip_index, trim_start, trim_end):
         trim_start=trim_start, trim_end=trim_end,
     )
     output(result, "Clip trimmed")
+    autosave_session_if_needed()
 
 
 @clip.command("split")
@@ -380,6 +398,7 @@ def clip_split(track_index, clip_index, split_time):
         sess.get_project(), track_index, clip_index, split_time,
     )
     output(result, f"Split clip into 2 parts at {split_time}s")
+    autosave_session_if_needed()
 
 
 @clip.command("move")
@@ -395,6 +414,7 @@ def clip_move(track_index, clip_index, new_start):
         sess.get_project(), track_index, clip_index, new_start,
     )
     output(result, f"Moved clip to {new_start}s")
+    autosave_session_if_needed()
 
 
 @clip.command("list")
@@ -455,6 +475,7 @@ def effect_add(name, track_index, param):
     sess.snapshot(f"Add effect {name} to track {track_index}")
     result = fx_mod.add_effect(sess.get_project(), name, track_index, params)
     output(result, f"Added effect: {name}")
+    autosave_session_if_needed()
 
 
 @effect_group.command("remove")
@@ -467,6 +488,7 @@ def effect_remove(effect_index, track_index):
     sess.snapshot(f"Remove effect {effect_index} from track {track_index}")
     result = fx_mod.remove_effect(sess.get_project(), effect_index, track_index)
     output(result, f"Removed effect {effect_index}")
+    autosave_session_if_needed()
 
 
 @effect_group.command("set")
@@ -486,6 +508,7 @@ def effect_set(effect_index, param, value, track_index):
     fx_mod.set_effect_param(sess.get_project(), effect_index, param, value, track_index)
     output({"effect": effect_index, "param": param, "value": value},
            f"Set effect {effect_index} {param} = {value}")
+    autosave_session_if_needed()
 
 
 @effect_group.command("list")
@@ -514,6 +537,7 @@ def selection_set(start, end):
     sess = get_session()
     result = sel_mod.set_selection(sess.get_project(), start, end)
     output(result, f"Selection: {start}s - {end}s")
+    autosave_session_if_needed()
 
 
 @selection.command("all")
@@ -523,6 +547,7 @@ def selection_all():
     sess = get_session()
     result = sel_mod.select_all(sess.get_project())
     output(result, "Selected all")
+    autosave_session_if_needed()
 
 
 @selection.command("none")
@@ -532,6 +557,7 @@ def selection_none():
     sess = get_session()
     result = sel_mod.select_none(sess.get_project())
     output(result, "Selection cleared")
+    autosave_session_if_needed()
 
 
 @selection.command("info")
@@ -561,6 +587,7 @@ def label_add(start, end, text):
     sess.snapshot(f"Add label at {start}")
     result = label_mod.add_label(sess.get_project(), start, end, text)
     output(result, f"Added label: {text or f'at {start}s'}")
+    autosave_session_if_needed()
 
 
 @label.command("remove")
@@ -572,6 +599,7 @@ def label_remove(index):
     sess.snapshot(f"Remove label {index}")
     removed = label_mod.remove_label(sess.get_project(), index)
     output(removed, f"Removed label: {removed.get('text', '')}")
+    autosave_session_if_needed()
 
 
 @label.command("list")
@@ -671,6 +699,7 @@ def session_undo():
     sess = get_session()
     desc = sess.undo()
     output({"undone": desc}, f"Undone: {desc}")
+    autosave_session_if_needed()
 
 
 @session_group.command("redo")
@@ -680,6 +709,7 @@ def session_redo():
     sess = get_session()
     desc = sess.redo()
     output({"redone": desc}, f"Redone: {desc}")
+    autosave_session_if_needed()
 
 
 @session_group.command("history")