From 7021c74f803097c6927f0fbdc2a8307f7faf6554 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Mon, 15 Jun 2026 15:31:16 -0600 Subject: [PATCH 01/10] feat(hooks): add local telemetry collection hooks --- .github/hooks/telemetry.json | 142 +++ .../hooks/telemetry/Invoke-TelemetryClean.ps1 | 87 ++ .../telemetry/Invoke-TelemetryCollector.ps1 | 80 ++ .github/hooks/telemetry/_telemetry_core.py | 1097 +++++++++++++++++ .github/hooks/telemetry/clean-telemetry.sh | 94 ++ .../telemetry/generate-telemetry-report.sh | 209 ++++ .github/hooks/telemetry/pyproject.toml | 27 + .github/hooks/telemetry/report.html | 808 ++++++++++++ .../hooks/telemetry/telemetry-collector.sh | 64 + .github/hooks/telemetry/tests/fuzz_harness.py | 129 ++ .../telemetry/tests/test_telemetry_core.py | 687 +++++++++++ .github/hooks/telemetry/uv.lock | 123 ++ .gitignore | 3 + collections/hve-core-all.collection.md | 6 + collections/hve-core-all.collection.yml | 2 + collections/hve-core.collection.md | 6 + collections/hve-core.collection.yml | 3 + docs/contributing/README.md | 2 + docs/contributing/hooks.md | 123 ++ docs/customization/README.md | 3 +- docs/customization/local-telemetry.md | 242 ++++ .../hve-core-all/.github/plugin/plugin.json | 3 +- plugins/hve-core-all/README.md | 6 + plugins/hve-core-all/hooks/telemetry | 1 + plugins/hve-core-all/hooks/telemetry.json | 142 +++ plugins/hve-core/.github/plugin/plugin.json | 3 +- plugins/hve-core/README.md | 6 + plugins/hve-core/hooks/telemetry | 1 + plugins/hve-core/hooks/telemetry.json | 142 +++ .../Modules/CollectionHelpers.psm1 | 44 +- .../schemas/collection-manifest.schema.json | 2 +- scripts/plugins/Generate-Plugins.ps1 | 8 +- scripts/plugins/Modules/PluginHelpers.psm1 | 139 ++- 33 files changed, 4414 insertions(+), 20 deletions(-) create mode 100644 .github/hooks/telemetry.json create mode 100644 .github/hooks/telemetry/Invoke-TelemetryClean.ps1 create mode 100644 .github/hooks/telemetry/Invoke-TelemetryCollector.ps1 create mode 100644 .github/hooks/telemetry/_telemetry_core.py create mode 100755 .github/hooks/telemetry/clean-telemetry.sh create mode 100755 .github/hooks/telemetry/generate-telemetry-report.sh create mode 100644 .github/hooks/telemetry/pyproject.toml create mode 100644 .github/hooks/telemetry/report.html create mode 100755 .github/hooks/telemetry/telemetry-collector.sh create mode 100644 .github/hooks/telemetry/tests/fuzz_harness.py create mode 100644 .github/hooks/telemetry/tests/test_telemetry_core.py create mode 100644 .github/hooks/telemetry/uv.lock create mode 100644 docs/contributing/hooks.md create mode 100644 docs/customization/local-telemetry.md create mode 120000 plugins/hve-core-all/hooks/telemetry create mode 100644 plugins/hve-core-all/hooks/telemetry.json create mode 120000 plugins/hve-core/hooks/telemetry create mode 100644 plugins/hve-core/hooks/telemetry.json diff --git a/.github/hooks/telemetry.json b/.github/hooks/telemetry.json new file mode 100644 index 000000000..74a205e3f --- /dev/null +++ b/.github/hooks/telemetry.json @@ -0,0 +1,142 @@ +{ + "version": 1, + "description": "Records Copilot session lifecycle events to local telemetry for reporting.", + "hooks": { + "sessionStart": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SessionStart": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "userPromptSubmitted": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "UserPromptSubmit": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preToolUse": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PreToolUse": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "postToolUse": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PostToolUse": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStart": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SubagentStart": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStop": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SubagentStop": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "agentStop": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "Stop": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "sessionEnd": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preCompact": [ + { + "type": "command", + "bash": ".github/hooks/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PreCompact": [ + { + "type": "command", + "command": ".github/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ] + } +} diff --git a/.github/hooks/telemetry/Invoke-TelemetryClean.ps1 b/.github/hooks/telemetry/Invoke-TelemetryClean.ps1 new file mode 100644 index 000000000..743b620e6 --- /dev/null +++ b/.github/hooks/telemetry/Invoke-TelemetryClean.ps1 @@ -0,0 +1,87 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +#Requires -Version 7.0 + +<# +.SYNOPSIS + Removes telemetry artifacts written by the Copilot telemetry hooks. +.DESCRIPTION + Delegates to the shared Python engine's clean mode. By default cleans this + project's telemetry store; -AllDirs extends the cleanup to every registered + project plus the user-level HVE home directory. This thin wrapper keeps the + cleanup logic in a single implementation (Python) shared with the bash entry + point clean-telemetry.sh. +.PARAMETER AllDirs + Also clean every per-project telemetry directory recorded in the user-level + registry, plus the generated launchers, report, and registry in the HVE home + directory. +.PARAMETER Path + Telemetry directory. Default: /.copilot-tracking/telemetry +.PARAMETER DryRun + List what would be removed without deleting anything. +.PARAMETER Force + Skip the confirmation prompt (required for non-interactive use). +.NOTES + Runs via: pwsh Invoke-TelemetryClean.ps1 +#> +[CmdletBinding(SupportsShouldProcess, ConfirmImpact = 'High')] +param( + [switch]$AllDirs, + [string]$Path, + [switch]$DryRun, + [switch]$Force +) + +$ErrorActionPreference = 'Stop' + +#region Resolve repo root +$RepoRoot = $env:HVE_REPO_ROOT +if (-not $RepoRoot -and (Get-Command git -ErrorAction SilentlyContinue)) { + try { $RepoRoot = & git -C $PSScriptRoot rev-parse --show-toplevel 2>$null } catch { $RepoRoot = $null } +} +if (-not $RepoRoot) { + $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +} +#endregion Resolve repo root + +# Require Python3 for the shared telemetry engine +$Python = Get-Command python3 -ErrorAction SilentlyContinue +if (-not $Python) { + $Python = Get-Command python -ErrorAction SilentlyContinue +} +if (-not $Python) { + Write-Error "'python3' is required but not installed" + exit 1 +} + +# Resolve the shared telemetry engine from the skill directory +$CorePy = Join-Path $PSScriptRoot '_telemetry_core.py' +if (-not (Test-Path $CorePy)) { + Write-Error "Telemetry engine not found: $CorePy" + exit 1 +} + +$TelemetryDir = if ($Path) { $Path } else { Join-Path $RepoRoot '.copilot-tracking/telemetry' } + +# Prompt before destructive deletion. Skipped on -DryRun (non-destructive) and +# bypassed with -Force (required for non-interactive use). +if (-not $DryRun -and -not $Force) { + $scope = if ($AllDirs) { + 'ALL registered telemetry stores plus the user-level HVE home directory' + } else { + $TelemetryDir + } + if (-not $PSCmdlet.ShouldProcess($scope, 'Permanently remove telemetry artifacts')) { + Write-Host 'Aborted.' + exit 0 + } +} + +# Build the clean-mode argument list mirroring the bash wrapper's flags. +$CliArgs = @('clean') +if ($AllDirs) { $CliArgs += '--all-dirs' } +if ($DryRun) { $CliArgs += '--dry-run' } + +$env:HVE_TELEMETRY_DIR = $TelemetryDir +& $Python.Source $CorePy @CliArgs diff --git a/.github/hooks/telemetry/Invoke-TelemetryCollector.ps1 b/.github/hooks/telemetry/Invoke-TelemetryCollector.ps1 new file mode 100644 index 000000000..8f6b12f52 --- /dev/null +++ b/.github/hooks/telemetry/Invoke-TelemetryCollector.ps1 @@ -0,0 +1,80 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +#Requires -Version 7.0 + +<# +.SYNOPSIS + Copilot hook handler that delegates telemetry collection to the shared Python engine. +.DESCRIPTION + Reads JSON from stdin for each hook lifecycle event, checks the opt-in gate, + and delegates all processing to _telemetry_core.py. This thin wrapper keeps + the collection logic in a single implementation (Python) shared with the bash + hook entry point. +.NOTES + Runs via: Copilot agent hook (stdin JSON, stdout JSON) +#> +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' + +#region Resolve repo root +$RepoRoot = $env:HVE_REPO_ROOT +if (-not $RepoRoot -and (Get-Command git -ErrorAction SilentlyContinue)) { + try { $RepoRoot = & git rev-parse --show-toplevel 2>$null } catch { $RepoRoot = $null } +} +if (-not $RepoRoot) { + $RepoRoot = '.' +} +#endregion Resolve repo root + +#region Opt-in gate +$Enabled = $env:HVE_TELEMETRY -eq '1' +if (-not $Enabled) { + $MarkerPath = Join-Path $RepoRoot '.hve-telemetry' + $Enabled = Test-Path $MarkerPath +} +if (-not $Enabled) { + '{"continue":true}' + return +} +#endregion Opt-in gate + +# Require Python3 for JSON processing +$Python = Get-Command python3 -ErrorAction SilentlyContinue +if (-not $Python) { + $Python = Get-Command python -ErrorAction SilentlyContinue +} +if (-not $Python) { + Write-Warning 'HVE telemetry enabled but python3 not found — events will not be recorded' + '{"continue":true}' + return +} + +# Resolve the shared telemetry engine from the skill directory +$CorePy = Join-Path $PSScriptRoot '_telemetry_core.py' + +if (-not (Test-Path $CorePy)) { + Write-Warning "Telemetry engine not found at $CorePy — events will not be recorded" + '{"continue":true}' + return +} + +$TelemetryDir = if ($env:HVE_TELEMETRY_DIR) { $env:HVE_TELEMETRY_DIR } else { Join-Path $RepoRoot '.copilot-tracking/telemetry' } +if (-not (Test-Path $TelemetryDir)) { + New-Item -ItemType Directory -Path $TelemetryDir -Force | Out-Null +} + +# Delegate all JSON processing to the shared Python telemetry engine +$RawInput = $input | Out-String +try { + $env:HVE_REPO_ROOT = $RepoRoot + $env:HVE_TELEMETRY_DIR = $TelemetryDir + $RawInput | & $Python.Source $CorePy collect 2>$null +} +catch { + Write-Verbose "Telemetry collection error: $_" +} + +'{"continue":true}' diff --git a/.github/hooks/telemetry/_telemetry_core.py b/.github/hooks/telemetry/_telemetry_core.py new file mode 100644 index 000000000..06494ea05 --- /dev/null +++ b/.github/hooks/telemetry/_telemetry_core.py @@ -0,0 +1,1097 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +"""Canonical telemetry engine shared by the Copilot hook collectors. + +This module is the single source of truth for telemetry collection. The bash +collector ``telemetry-collector.sh`` invokes the ``collect`` mode to record one +hook event (and enrich the session at ``Stop``), while +``generate-telemetry-report.sh`` invokes the ``aggregate-debug``, +``aggregate-session``, and ``list-dirs`` modes to join model/token data and +discover per-project telemetry stores for reports. ``clean-telemetry.sh`` +invokes the ``clean`` mode to remove telemetry artifacts from one or every +registered store. + +The PowerShell collector ``Invoke-TelemetryCollector.ps1`` is a thin wrapper +that delegates to this same engine via ``collect``, so the collection logic +stays single-sourced across platforms. +""" + +from __future__ import annotations + +import datetime +import glob +import json +import os +import shlex +import shutil +import sys +from collections.abc import Iterable, Iterator +from pathlib import Path + + +def _detect_client() -> str: + """Infer the Copilot surface that invoked this hook.""" + if os.environ.get("GITHUB_COPILOT_API_TOKEN"): + return "cloud-agent" + if os.environ.get("VSCODE_PID") or os.environ.get("VSCODE_IPC_HOOK_CLI"): + return "vscode" + return "cli" + + +EVENT_ALIASES = { + "sessionStart": "SessionStart", + "userPromptSubmitted": "UserPromptSubmit", + "preToolUse": "PreToolUse", + "postToolUse": "PostToolUse", + "subagentStart": "SubagentStart", + "subagentStop": "SubagentStop", + "agentStop": "Stop", + "sessionEnd": "Stop", + "preCompact": "PreCompact", +} + + +def iter_jsonl(path: str | os.PathLike[str]) -> Iterator[dict]: + """Yield each well-formed JSON object from a JSONL file, skipping junk.""" + try: + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except ValueError: + continue + if isinstance(obj, dict): + yield obj + except OSError: + return + + +def _is_safe_sid(sid: str) -> bool: + """Return True when a session id is safe to embed in a filesystem path. + + Rejects empty ids and any value containing a path separator or ``..`` + traversal sequence so callers never build paths outside their store. + """ + return bool(sid) and not ( + os.sep in sid or "/" in sid or "\\" in sid or ".." in sid + ) + + +def collect_sids(hook_files: Iterable[str]) -> set[str]: + """Collect every safe session id referenced across the given hook files.""" + sids: set[str] = set() + for hook_file in hook_files: + for obj in iter_jsonl(hook_file): + sid = obj.get("sid") + if sid and _is_safe_sid(sid): + sids.add(sid) + return sids + + +def copilot_home() -> Path: + """Return the Copilot home directory, honoring ``COPILOT_HOME``.""" + override = os.environ.get("COPILOT_HOME") + return Path(override) if override else Path.home() / ".copilot" + + +def hve_home() -> Path: + """Return the HVE user-level directory, honoring ``HVE_HOME``.""" + override = os.environ.get("HVE_HOME") + return Path(override) if override else Path.home() / ".hve" + + +def telemetry_registry() -> Path: + """Return the user-level registry that lists per-project telemetry dirs.""" + return hve_home() / "telemetry-dirs.txt" + + +def read_registry_dirs(registry: Path | None = None) -> list[str]: + """Return registered telemetry directories in order, de-duplicated.""" + registry = registry if registry is not None else telemetry_registry() + try: + text = registry.read_text(encoding="utf-8") + except OSError: + return [] + dirs: list[str] = [] + seen: set[str] = set() + for line in text.splitlines(): + entry = line.strip() + if entry and entry not in seen: + seen.add(entry) + dirs.append(entry) + return dirs + + +def register_telemetry_dir(tel_dir: Path, registry: Path | None = None) -> None: + """Record an absolute telemetry directory in the user-level registry. + + Lets the report tooling discover every per-project telemetry store without + relying on environment propagation. Resolves to an absolute path and skips + the append when already present so the registry stays de-duplicated. + """ + registry = registry if registry is not None else telemetry_registry() + try: + resolved = str(tel_dir.resolve()) + except OSError: + resolved = os.path.abspath(str(tel_dir)) + if resolved in read_registry_dirs(registry): + return + try: + registry.parent.mkdir(parents=True, exist_ok=True) + with open(registry, "a", encoding="utf-8") as handle: + handle.write(resolved + "\n") + except OSError: + return + + +_BASH_LAUNCHER = """#!/usr/bin/env bash +# Generated by HVE telemetry. Regenerated each session; edits will be lost. +# Cross-project telemetry report launcher. Lives in the HVE home directory +# alongside the registry of per-project telemetry stores, so it does not need +# the (version-pinned) extension install path. Run from this directory: +# ./generate-report.sh # today, every project +# ./generate-report.sh --date all # every captured day, every project +REPORT_SCRIPT=__REPORT_SCRIPT__ +if [[ ! -f "$REPORT_SCRIPT" ]]; then + echo "Telemetry report script not found: $REPORT_SCRIPT" >&2 + echo "Start a new Copilot session to regenerate this launcher." >&2 + exit 1 +fi +exec bash "$REPORT_SCRIPT" --all-dirs --output __OUT__ "$@" +""" + +_PWSH_LAUNCHER = """# Generated by HVE telemetry. Regenerated each session; edits will be lost. +# Cross-project telemetry report launcher. Lives in the HVE home directory +# alongside the registry of per-project telemetry stores. Requires bash (for +# example Git Bash). Run from this directory: +# ./generate-report.ps1 # today, every project +# ./generate-report.ps1 --date all # every captured day, every project +$ReportScript = '__REPORT_SCRIPT__' +if (-not (Test-Path $ReportScript)) { + Write-Error "Telemetry report script not found: $ReportScript" + Write-Error 'Start a new Copilot session to regenerate this launcher.' + exit 1 +} +bash $ReportScript --all-dirs --output '__OUT__' @args +""" + + +_BASH_CLEAN_LAUNCHER = """#!/usr/bin/env bash +# Generated by HVE telemetry. Regenerated each session; edits will be lost. +# Cross-project telemetry cleanup launcher. Removes telemetry artifacts from +# every registered per-project store and this HVE home directory. Run from +# this directory: +# ./clean-telemetry.sh # remove all telemetry artifacts +# ./clean-telemetry.sh --dry-run # list what would be removed +CLEAN_SCRIPT=__CLEAN_SCRIPT__ +if [[ ! -f "$CLEAN_SCRIPT" ]]; then + echo "Telemetry clean script not found: $CLEAN_SCRIPT" >&2 + echo "Start a new Copilot session to regenerate this launcher." >&2 + exit 1 +fi +exec bash "$CLEAN_SCRIPT" --all-dirs "$@" +""" + +_PWSH_CLEAN_LAUNCHER = """\ +# Generated by HVE telemetry. Regenerated each session; edits will be lost. +# Cross-project telemetry cleanup launcher. Removes telemetry artifacts from +# every registered per-project store and this HVE home directory. Runs natively +# through PowerShell. Run from this directory: +# ./clean-telemetry.ps1 # remove all telemetry artifacts +# ./clean-telemetry.ps1 -DryRun # list what would be removed +$CleanScript = '__CLEAN_PS1__' +if (-not (Test-Path $CleanScript)) { + Write-Error "Telemetry clean script not found: $CleanScript" + Write-Error 'Start a new Copilot session to regenerate this launcher.' + exit 1 +} +& $CleanScript -AllDirs @args +""" + + +def _is_windows() -> bool: + """Return True when running on Windows. + + Factored out so launcher generation can pick the native interpreter and so + tests can exercise both platform branches. + """ + return os.name == "nt" + + +def write_report_launchers(script_dir: Path | None = None) -> None: + """Emit cross-project report and cleanup launchers into the HVE home dir. + + Extension users lack the repository's launcher scripts and cannot easily + locate the version-pinned extension install path. These launchers live next + to the registry in the HVE home directory, are refreshed each session, and + delegate to the canonical report generator and cleanup wrappers in + cross-project mode so running them spans every project. + + Only the launchers for the host platform are written: PowerShell (``.ps1``) + on Windows, POSIX shell (``.sh``) elsewhere. The cleanup launcher is fully + native per platform (bash wrapper on POSIX, PowerShell wrapper on Windows), + so no cross-interpreter dependency is required for cleanup. + """ + if script_dir is None: + script_dir = Path(__file__).resolve().parent + report_script = str(script_dir / "generate-telemetry-report.sh") + clean_script = str(script_dir / "clean-telemetry.sh") + clean_ps1 = str(script_dir / "Invoke-TelemetryClean.ps1") + home = hve_home() + out_path = str(home / "report.generated.html") + try: + home.mkdir(parents=True, exist_ok=True) + if _is_windows(): + # Quote for PowerShell single-quoted strings (double embedded '). + pwsh_text = _PWSH_LAUNCHER.replace( + "__REPORT_SCRIPT__", report_script.replace("'", "''") + ).replace("__OUT__", out_path.replace("'", "''")) + pwsh_clean_text = _PWSH_CLEAN_LAUNCHER.replace( + "__CLEAN_PS1__", clean_ps1.replace("'", "''") + ) + (home / "generate-report.ps1").write_text(pwsh_text, encoding="utf-8") + (home / "clean-telemetry.ps1").write_text( + pwsh_clean_text, encoding="utf-8" + ) + else: + # Shell-quote so unusual install paths (spaces, quotes, ``$``) + # cannot break or inject into the generated launchers. + bash_text = _BASH_LAUNCHER.replace( + "__REPORT_SCRIPT__", shlex.quote(report_script) + ).replace("__OUT__", shlex.quote(out_path)) + bash_clean_text = _BASH_CLEAN_LAUNCHER.replace( + "__CLEAN_SCRIPT__", shlex.quote(clean_script) + ) + sh_path = home / "generate-report.sh" + sh_path.write_text(bash_text, encoding="utf-8") + sh_path.chmod(0o755) + clean_sh_path = home / "clean-telemetry.sh" + clean_sh_path.write_text(bash_clean_text, encoding="utf-8") + clean_sh_path.chmod(0o755) + except OSError: + return + + +def find_process_log(state_dir: Path, home: Path) -> str | None: + """Locate the CLI process log via the session lock file PID.""" + pid = None + try: + for lock_file in os.listdir(state_dir): + if lock_file.startswith("inuse.") and lock_file.endswith(".lock"): + pid = lock_file.split(".")[1] + break + except OSError: + return None + if not pid: + return None + candidates = glob.glob(str(home / "logs" / f"process-*-{pid}.log")) + return candidates[0] if candidates else None + + +def _log_references_interactions( + log_path: str, interaction_ids: set[str] +) -> bool: + """Return True when a process log references any of the interaction ids. + + Uses a cheap line substring scan so logs that cannot belong to the session + are rejected without a full JSON parse. + """ + try: + with open(log_path, encoding="utf-8", errors="replace") as handle: + for line in handle: + if "interaction_id" in line and any( + iid in line for iid in interaction_ids + ): + return True + except OSError: + return False + return False + + +def find_process_logs_for_session( + state_dir: Path, home: Path, interaction_ids: set[str] +) -> list[str]: + """Return the process logs that hold usage for a session. + + Prefers the log named after the live session lock PID. When that lock is + gone (the session has ended), falls back to scanning every process log for + one whose entries reference this session's interaction ids, so per-request + input token data survives past session end rather than degrading to the + compaction-only state fallback. + """ + locked = find_process_log(state_dir, home) + if locked: + return [locked] + if not interaction_ids: + return [] + matches: list[str] = [] + for path in sorted(glob.glob(str(home / "logs" / "process-*.log"))): + if _log_references_interactions(path, interaction_ids): + matches.append(path) + return matches + + +def parse_process_log(log_path: str, interaction_ids: set[str]) -> list[dict]: + """Parse assistant_usage blocks from a process log, filtered by id.""" + results: list[dict] = [] + # Process logs use brace-delimited JSON blocks (one top-level '{' … '}' per + # entry) rather than newline-delimited JSON, so we accumulate lines between + # matching braces and only parse blocks containing assistant_usage data. + in_block = False + block_lines: list[str] = [] + block_has_usage = False + try: + with open(log_path, encoding="utf-8") as handle: + for line in handle: + stripped = line.rstrip() + if stripped == "{": + in_block = True + block_lines = [stripped] + block_has_usage = False + elif in_block: + block_lines.append(stripped) + if '"assistant_usage"' in stripped: + block_has_usage = True + if stripped == "}": + if block_has_usage: + try: + obj = json.loads("\n".join(block_lines)) + except ValueError: + obj = None + if obj and obj.get("kind") == "assistant_usage": + props = obj.get("properties", {}) + if props.get("interaction_id", "") in interaction_ids: + results.append(obj) + in_block = False + block_lines = [] + except OSError: + return results + return results + + +def scan_session_state(state_file: str | os.PathLike[str]) -> dict: + """Read events.jsonl once for session metadata and interaction ids.""" + interaction_ids: set[str] = set() + models: dict[str, int] = {} + subagent_map: dict[str, str] = {} + messages = 0 + turns = 0 + reasoning_effort = "" + first_ts = "" + last_ts = "" + for evt in iter_jsonl(state_file): + data = evt.get("data", {}) + if not isinstance(data, dict): + continue + ts = evt.get("timestamp", "") + if ts: + if not first_ts or ts < first_ts: + first_ts = ts + if not last_ts or ts > last_ts: + last_ts = ts + etype = evt.get("type", "") + if etype == "assistant.message": + messages += 1 + model = data.get("model", "") + if model: + models[model] = models.get(model, 0) + 1 + iid = data.get("interactionId", "") + if iid: + interaction_ids.add(iid) + elif etype == "assistant.turn_start": + turns += 1 + iid = data.get("interactionId", "") + if iid: + interaction_ids.add(iid) + elif etype == "session.model_change": + reasoning_effort = data.get("reasoningEffort", "") + elif etype == "subagent.started": + tcid = data.get("toolCallId", "") + aname = data.get("agentName", "") or data.get("agentDisplayName", "") + if tcid and aname: + subagent_map[tcid] = aname + return { + "interaction_ids": interaction_ids, + "models": models, + "subagent_map": subagent_map, + "messages": messages, + "turns": turns, + "reasoning_effort": reasoning_effort, + "first_ts": first_ts, + "last_ts": last_ts, + } + + +def _totals_from_process_log(entries: list[dict]) -> dict: + """Accumulate token totals and per-model usage from process-log entries.""" + total_input = total_output = cache_read = cache_write = total_nano_aiu = 0 + total_input_uncached = 0 + model_usage: dict[str, dict] = {} + for entry in entries: + props = entry.get("properties", {}) + metrics = entry.get("metrics", {}) + model = props.get("model", "unknown") + total_input += metrics.get("input_tokens", 0) + total_input_uncached += metrics.get("input_tokens_uncached", 0) + total_output += metrics.get("output_tokens", 0) + cache_read += metrics.get("cache_read_tokens", 0) + cache_write += metrics.get("cache_write_tokens", 0) + total_nano_aiu += metrics.get("total_nano_aiu", 0) + bucket = model_usage.setdefault( + model, + { + "output_tokens": 0, + "messages": 0, + "input_tokens": 0, + "input_tokens_uncached": 0, + }, + ) + bucket["output_tokens"] += metrics.get("output_tokens", 0) + bucket["input_tokens"] += metrics.get("input_tokens", 0) + bucket["input_tokens_uncached"] += metrics.get("input_tokens_uncached", 0) + bucket["messages"] += 1 + return { + "output_tokens": total_output, + "input_tokens": total_input, + "input_tokens_uncached": total_input_uncached, + "cache_read_tokens": cache_read, + "cache_write_tokens": cache_write, + "total_nano_aiu": total_nano_aiu, + "model_usage": model_usage, + } + + +def _per_agent_usage_from_process_log( + entries: list[dict], subagent_map: dict[str, str] +) -> dict[str, dict]: + """Partition process-log entries by agent_id and compute per-agent totals. + + Returns a dict keyed by agent display name with token usage per subagent. + Only includes agents that have at least one request attributed to them. + The root agent (entries without agent_id or with initiator != "sub-agent") + is keyed as "root". + """ + agent_entries: dict[str, list[dict]] = {} + for entry in entries: + props = entry.get("properties", {}) + if props.get("initiator") == "sub-agent": + agent_id = props.get("agent_id", "") + label = subagent_map.get(agent_id, agent_id or "sub-agent") + else: + label = "root" + agent_entries.setdefault(label, []).append(entry) + + result: dict[str, dict] = {} + for label, agent_list in agent_entries.items(): + totals = _totals_from_process_log(agent_list) + result[label] = { + "output_tokens": totals["output_tokens"], + "input_tokens": totals["input_tokens"], + "input_tokens_uncached": totals.get("input_tokens_uncached", 0), + "cache_read_tokens": totals["cache_read_tokens"], + "cache_write_tokens": totals["cache_write_tokens"], + "total_nano_aiu": totals["total_nano_aiu"], + "requests": sum(1 for _ in agent_list), + } + return result + + +def _new_usage_bucket() -> dict: + """Return a fresh per-model usage bucket with the unified key shape.""" + return { + "output_tokens": 0, + "messages": 0, + "input_tokens": 0, + "input_tokens_uncached": 0, + } + + +def _totals_from_state_fallback(state_file: str | os.PathLike[str]) -> dict: + """Approximate token totals from events.jsonl when no process log exists. + + Sums per-model ``session.shutdown`` ``modelMetrics`` across every shutdown + in the file. A resumed session writes one shutdown per run segment with + counters that reset on each resume, so the segments must be summed to + recover whole-session totals. ``usage.inputTokens`` already includes cache + reads and writes, so fresh (uncached) input is recovered by subtracting + them. + + Output is reconciled against the ``assistant.message`` per-message sum, + which stays complete even when a run segment ends without ``modelMetrics`` + (an aborted or minimal shutdown) or emits subagent output under no tracked + model. The larger of the two sources wins so output is never undercounted. + + When no shutdown exists yet (a live session that has not ended a segment), + only per-message output is known; input, cache, and AIU are reported as + ``None`` so the report can distinguish "unknown" from a true zero. + """ + total_input = shutdown_output = cache_read = cache_write = total_nano_aiu = 0 + total_input_uncached = 0 + model_usage: dict[str, dict] = {} + msg_output_total = 0 + msg_output_by_model: dict[str, int] = {} + had_shutdown = False + for evt in iter_jsonl(state_file): + data = evt.get("data", {}) + if not isinstance(data, dict): + continue + etype = evt.get("type", "") + if etype == "assistant.message": + output_tokens = data.get("outputTokens", 0) + if output_tokens: + msg_output_total += output_tokens + model = data.get("model", "") + if model: + msg_output_by_model[model] = ( + msg_output_by_model.get(model, 0) + output_tokens + ) + continue + if etype != "session.shutdown": + continue + metrics = data.get("modelMetrics", {}) + if not isinstance(metrics, dict): + continue + had_shutdown = True + for model, m in metrics.items(): + if not isinstance(m, dict): + continue + usage = m.get("usage", {}) + in_tok = usage.get("inputTokens", 0) + out_tok = usage.get("outputTokens", 0) + cr = usage.get("cacheReadTokens", 0) + cw = usage.get("cacheWriteTokens", 0) + uncached = max(in_tok - cr - cw, 0) + requests = m.get("requests", {}).get("count", 0) + total_input += in_tok + shutdown_output += out_tok + cache_read += cr + cache_write += cw + total_input_uncached += uncached + total_nano_aiu += m.get("totalNanoAiu", 0) + bucket = model_usage.setdefault(model, _new_usage_bucket()) + bucket["output_tokens"] += out_tok + bucket["input_tokens"] += in_tok + bucket["input_tokens_uncached"] += uncached + bucket["messages"] += requests + if had_shutdown: + # Reconcile output per model and in total against the message sum, + # which is complete even when a segment lacked modelMetrics. + for model, out_tok in msg_output_by_model.items(): + bucket = model_usage.setdefault(model, _new_usage_bucket()) + if out_tok > bucket["output_tokens"]: + bucket["output_tokens"] = out_tok + return { + "output_tokens": max(shutdown_output, msg_output_total), + "input_tokens": total_input, + "input_tokens_uncached": total_input_uncached, + "cache_read_tokens": cache_read, + "cache_write_tokens": cache_write, + "total_nano_aiu": total_nano_aiu, + "model_usage": model_usage, + } + # Live session with no completed segment: only per-message output is known. + for model, out_tok in msg_output_by_model.items(): + model_usage.setdefault(model, _new_usage_bucket())["output_tokens"] += out_tok + return { + "output_tokens": msg_output_total, + "input_tokens": None, + "input_tokens_uncached": None, + "cache_read_tokens": None, + "cache_write_tokens": None, + "total_nano_aiu": None, + "model_usage": model_usage, + } + + +def build_session_summary( + sid: str, + state_dir: Path, + state_file: str | os.PathLike[str], + home: Path, + ts_override: str | None = None, + client: str = "", +) -> dict: + """Build a SessionSummary event for a session. + + Prefers precise per-request metrics from the CLI process log and falls + back to summed ``session.shutdown`` metrics in events.jsonl when the + process log is unavailable. The inner readers each swallow their own + ``OSError`` and yield empty data, so a summary is produced even when the + underlying files are unreadable. + """ + meta = scan_session_state(state_file) + interaction_ids = meta["interaction_ids"] + process_logs = find_process_logs_for_session(state_dir, home, interaction_ids) + totals = None + agent_usage: dict[str, dict] | None = None + token_source = "state_fallback" + if process_logs and interaction_ids: + entries: list[dict] = [] + for log in process_logs: + entries.extend(parse_process_log(log, interaction_ids)) + if entries: + totals = _totals_from_process_log(entries) + token_source = "process_log" + # Compute per-subagent token attribution when subagents were used. + if meta["subagent_map"]: + agent_usage = _per_agent_usage_from_process_log( + entries, meta["subagent_map"] + ) + if totals is None: + totals = _totals_from_state_fallback(state_file) + + summary = { + "ts": ts_override if ts_override is not None else meta["last_ts"], + "sid": sid, + "event": "SessionSummary", + "first_ts": meta["first_ts"], + "last_ts": meta["last_ts"], + "models": meta["models"], + "model_usage": totals["model_usage"], + "output_tokens": totals["output_tokens"], + "input_tokens": totals["input_tokens"], + "cache_read_tokens": totals["cache_read_tokens"], + "cache_write_tokens": totals["cache_write_tokens"], + "total_nano_aiu": totals["total_nano_aiu"], + "token_source": token_source, + "turns": meta["turns"], + "messages": meta["messages"], + } + # Fresh (uncached) input is available from the process log and from summed + # session.shutdown metrics; omit the key only when the fallback found no + # shutdown (None) so the report can distinguish "unknown" from a true zero. + uncached = totals.get("input_tokens_uncached") + if uncached is not None: + summary["input_tokens_uncached"] = uncached + if meta["reasoning_effort"]: + summary["reasoning_effort"] = meta["reasoning_effort"] + if meta["subagent_map"]: + summary["subagent_map"] = meta["subagent_map"] + if agent_usage: + summary["agent_usage"] = agent_usage + if client: + summary["client"] = client + return summary + + +def _normalize_event(data: dict) -> str: + """Resolve the canonical PascalCase event name from a hook payload.""" + event = data.get("hook_event_name", "unknown") + if event == "unknown": + for key in ("hookEventName", "event"): + value = data.get(key) + if value and value != "unknown": + event = value + break + return EVENT_ALIASES.get(event, event) + + +def _normalize_timestamp(raw_ts: object) -> str: + """Coerce a hook timestamp (epoch ms or string) to an ISO-8601 string.""" + if isinstance(raw_ts, (int, float)): + return datetime.datetime.fromtimestamp( + raw_ts / 1000, tz=datetime.UTC + ).isoformat() + if isinstance(raw_ts, str) and raw_ts: + return raw_ts + return datetime.datetime.now(datetime.UTC).isoformat() + + +def _token_estimate(path: str) -> int: + """Estimate token count as ceil(file_size / 4). + + Uses the common ~4 chars-per-token heuristic for LLM token budgets. + """ + try: + # Ceiling division without importing math. + return int(-(-os.path.getsize(path) // 4)) + except OSError: + return 0 + + +class _AgentStack: + """Per-session agent stack persisted as a JSON array on disk. + + Tracks the active agent (root vs subagent) so telemetry entries can + attribute tool calls to the correct agent context. + """ + + def __init__(self, stack_dir: Path, sid: str) -> None: + self.stack_dir = stack_dir + self.stack_file = ( + stack_dir / f"{sid}.json" if _is_safe_sid(sid) else None + ) + + def _read(self) -> list[str]: + if self.stack_file and self.stack_file.exists(): + try: + with open(self.stack_file, encoding="utf-8") as handle: + data = json.load(handle) + if isinstance(data, list): + return data + except (OSError, ValueError): + return [] + return [] + + def current(self) -> str: + stack = self._read() + return stack[-1] if stack else "root" + + def push(self, name: str) -> None: + if not self.stack_file: + return + self.stack_dir.mkdir(parents=True, exist_ok=True) + stack = self._read() + stack.append(name) + with open(self.stack_file, "w", encoding="utf-8") as handle: + json.dump(stack, handle) + + def pop(self) -> None: + """Remove the topmost agent. Deletes the file when the stack empties.""" + if not self.stack_file or not self.stack_file.exists(): + return + stack = self._read() + if len(stack) > 1: + with open(self.stack_file, "w", encoding="utf-8") as handle: + json.dump(stack[:-1], handle) + else: + self.stack_file.unlink(missing_ok=True) + + def clear(self) -> None: + if self.stack_file and self.stack_file.exists(): + self.stack_file.unlink(missing_ok=True) + + +def build_entry(data: dict, event: str, stack: _AgentStack) -> dict | None: + """Build the JSONL telemetry entry for a single hook event. + + Returns ``None`` for unrecognized events, which the caller drops. + """ + sid = data.get("session_id") or data.get("sessionId", "") + cwd = data.get("cwd", os.getcwd()) + ts = _normalize_timestamp(data.get("timestamp", "")) + tool_name = data.get("tool_name") or data.get("toolName", "") + tool_input = data.get("tool_input") or data.get("toolArgs", {}) + tool_result = data.get("tool_result") or data.get("toolResult", "") + + if event == "unknown": + return None + + entry: dict = {"ts": ts, "sid": sid, "event": event, "cwd": cwd} + + if event == "SessionStart": + entry["source"] = data.get("source", "") + entry["client"] = _detect_client() + elif event == "UserPromptSubmit": + entry["prompt"] = (data.get("prompt", "") or "")[:200] + elif event == "PreToolUse": + entry["tool"] = tool_name + entry["tool_input_keys"] = ( + list(tool_input.keys()) if isinstance(tool_input, dict) else [] + ) + entry["agent"] = stack.current() + # Detect instructions and skills by file path convention to track + # which artifacts the agent loaded during the session. + fpath = tool_input.get("filePath") if isinstance(tool_input, dict) else None + if isinstance(fpath, str): + if fpath.endswith(".instructions.md"): + entry["instruction"] = fpath.split("/")[-1] + entry["tokens"] = _token_estimate(fpath) + elif fpath.endswith("SKILL.md"): + parts = fpath.rstrip("/").split("/") + idx = next((i for i, p in enumerate(parts) if p == "skills"), -1) + if idx >= 0 and idx + 2 < len(parts): + entry["skill"] = parts[idx + 2] + elif len(parts) >= 2: + entry["skill"] = parts[-2] + entry["tokens"] = _token_estimate(fpath) + if isinstance(tool_input, dict) and tool_name in ("runSubagent", "task"): + entry["subagent"] = ( + tool_input.get("agentName") + or tool_input.get("agent_type") + or tool_input.get("description", "") + ) + elif event == "PostToolUse": + entry["tool"] = tool_name + if isinstance(tool_result, dict): + text = tool_result.get("text_result_for_llm") or tool_result.get( + "textResultForLlm", "" + ) + entry["tool_response_len"] = len(text if isinstance(text, str) else str(text)) + elif isinstance(tool_result, str): + entry["tool_response_len"] = len(tool_result) + else: + entry["tool_response_len"] = len(json.dumps(tool_result)) + entry["agent"] = stack.current() + elif event == "SubagentStart": + agent_name = data.get("agent_name") or data.get("agentName", "") + entry["agent_name"] = agent_name + entry["agent_display_name"] = data.get("agent_display_name") or data.get( + "agentDisplayName", "" + ) + stack.push(agent_name) + elif event == "SubagentStop": + agent_name = data.get("agent_name") or data.get("agentName", "") + entry["agent_name"] = agent_name + stack.pop() + elif event == "PreCompact": + entry["trigger"] = data.get("trigger", "") + elif event == "Stop": + entry["stop_reason"] = data.get("stop_reason") or data.get("stopReason", "") + stack.clear() + + return entry + + +def _mode_collect() -> int: + """Process a single hook event from stdin; returns a process exit code.""" + try: + data = json.load(sys.stdin) + except ValueError: + return 0 + if not isinstance(data, dict): + return 0 + + sid = data.get("session_id") or data.get("sessionId", "") + # Reject session IDs containing path separators or traversal sequences + # to prevent writes outside the telemetry directory. + if sid and not _is_safe_sid(sid): + return 0 + + event = _normalize_event(data) + tel_dir = Path(os.environ.get("HVE_TELEMETRY_DIR", ".copilot-tracking/telemetry")) + date_str = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d") + log_file = tel_dir / f"sessions-{date_str}.jsonl" + stack_dir = tel_dir / ".stacks" + stack = _AgentStack(stack_dir, sid) + + entry = build_entry(data, event, stack) + if entry is None: + return 0 + + tel_dir.mkdir(parents=True, exist_ok=True) + # Record this project's telemetry dir once per session so cross-project + # reports can discover every store, and refresh the user-level launcher. + # SessionStart keeps these writes infrequent. + if event == "SessionStart": + register_telemetry_dir(tel_dir) + write_report_launchers() + with open(log_file, "a", encoding="utf-8") as handle: + handle.write(json.dumps(entry) + "\n") + + # Enrich the log with a SessionSummary of token totals and model usage. + # Emitted at Stop (session end) and at PreCompact: process logs rotate + # aggressively, so capturing a snapshot before compaction preserves + # per-request input data that would otherwise degrade to the + # compaction-only state fallback once the log is gone. Multiple summaries + # per session are expected; the report replaces by provenance rank and + # freshness rather than accumulating, so snapshots never double-count. + if event in ("Stop", "PreCompact") and sid: + home = copilot_home() + state_dir = home / "session-state" / sid + state_file = state_dir / "events.jsonl" + if state_file.is_file(): + summary = build_session_summary( + sid, state_dir, state_file, home, + ts_override=entry["ts"], client=_detect_client(), + ) + if summary is not None: + with open(log_file, "a", encoding="utf-8") as handle: + handle.write(json.dumps(summary) + "\n") + return 0 + + +def _mode_aggregate_debug(out: str, hook_files: list[str]) -> int: + """Emit llm_request events from VS Code debug logs for collected sids.""" + sids = collect_sids(hook_files) + if not sids: + return 1 + + home = Path.home() + patterns = [ + str(home / d / "data/User/workspaceStorage/**/debug-logs/**/*.jsonl") + for d in (".vscode-server-insiders", ".vscode-server", ".vscode") + ] + count = 0 + with open(out, "w", encoding="utf-8") as writer: + for pattern in patterns: + for path in glob.glob(pattern, recursive=True): + for obj in iter_jsonl(path): + if obj.get("type") == "llm_request" and obj.get("sid") in sids: + writer.write(json.dumps(obj) + "\n") + count += 1 + return 0 if count else 1 + + +def _mode_aggregate_session(out: str, hook_files: list[str]) -> int: + """Emit SessionSummary events from CLI session state for collected sids.""" + sids = collect_sids(hook_files) + if not sids: + return 1 + + home = copilot_home() + state_base = home / "session-state" + count = 0 + with open(out, "w", encoding="utf-8") as writer: + for sid in sids: + state_dir = state_base / sid + state_file = state_dir / "events.jsonl" + if not state_file.is_file(): + continue + summary = build_session_summary(sid, state_dir, state_file, home, client="cli") + if summary is not None: + writer.write(json.dumps(summary) + "\n") + count += 1 + return 0 if count else 1 + + +# Telemetry artifacts written into a per-project store. Cleanup targets only +# these known names so a directory a user pointed ``HVE_TELEMETRY_DIR`` at is +# never removed wholesale. +_TELEMETRY_FILE_ARTIFACTS = ("raw-input.jsonl", "report.generated.html") +_TELEMETRY_GLOB_ARTIFACTS = ("sessions-*.jsonl",) +_TELEMETRY_DIR_ARTIFACTS = (".stacks",) + +# Artifacts written into the HVE home directory (registry plus generated +# cross-project launchers and report). +_HVE_HOME_ARTIFACTS = ( + "telemetry-dirs.txt", + "report.generated.html", + "generate-report.sh", + "generate-report.ps1", + "clean-telemetry.sh", + "clean-telemetry.ps1", +) + + +def _remove_path(path: Path, dry_run: bool, removed: list[str]) -> None: + """Remove a file or directory, recording the deleted path. + + Missing paths and removal errors are ignored so cleanup is best-effort and + never aborts partway. + """ + if not path.exists() and not path.is_symlink(): + return + if dry_run: + removed.append(str(path)) + return + try: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + except OSError: + return + removed.append(str(path)) + + +def clean_telemetry_dir(tel_dir: Path, dry_run: bool, removed: list[str]) -> None: + """Remove known telemetry artifacts from a single per-project store.""" + if not tel_dir.is_dir(): + return + for name in _TELEMETRY_FILE_ARTIFACTS: + _remove_path(tel_dir / name, dry_run, removed) + for pattern in _TELEMETRY_GLOB_ARTIFACTS: + for match in sorted(tel_dir.glob(pattern)): + _remove_path(match, dry_run, removed) + for name in _TELEMETRY_DIR_ARTIFACTS: + _remove_path(tel_dir / name, dry_run, removed) + + +def _mode_clean(all_dirs: bool, dry_run: bool) -> int: + """Remove telemetry artifacts from the current store. + + With ``all_dirs`` the scope expands to every registered store plus the + generated launchers, report, and registry in the HVE home directory. + """ + removed: list[str] = [] + targets: list[Path] = [] + if all_dirs: + targets.extend(Path(d) for d in read_registry_dirs()) + targets.append( + Path(os.environ.get("HVE_TELEMETRY_DIR", ".copilot-tracking/telemetry")) + ) + + seen: set[str] = set() + for tel_dir in targets: + try: + key = str(tel_dir.resolve()) + except OSError: + key = str(tel_dir) + if key in seen: + continue + seen.add(key) + clean_telemetry_dir(tel_dir, dry_run, removed) + + if all_dirs: + home = hve_home() + for name in _HVE_HOME_ARTIFACTS: + _remove_path(home / name, dry_run, removed) + + verb = "Would remove" if dry_run else "Removed" + if removed: + for item in removed: + sys.stdout.write(f"{verb}: {item}\n") + sys.stdout.write(f"{verb} {len(removed)} item(s).\n") + else: + sys.stdout.write("No telemetry artifacts found to remove.\n") + return 0 + + +def _mode_list_dirs() -> int: + """Print registered telemetry dirs that still exist; prune dead entries. + + Pruning rewrites the registry only when stale paths are dropped, keeping + the cross-project report scan fast as repositories come and go. + """ + registry = telemetry_registry() + dirs = read_registry_dirs(registry) + live = [d for d in dirs if Path(d).is_dir()] + if live != dirs: + try: + registry.parent.mkdir(parents=True, exist_ok=True) + with open(registry, "w", encoding="utf-8") as handle: + handle.write("".join(d + "\n" for d in live)) + except OSError: + pass + for directory in live: + sys.stdout.write(directory + "\n") + return 0 + + +def main(argv: list[str]) -> int: + """Dispatch a CLI mode. See module docstring for the contract.""" + if not argv: + sys.stderr.write( + "usage: _telemetry_core.py " + " ...\n" + ) + return 2 + mode = argv[0] + if mode == "collect": + return _mode_collect() + if mode == "aggregate-debug": + if len(argv) < 2: + return 2 + return _mode_aggregate_debug(argv[1], argv[2:]) + if mode == "aggregate-session": + if len(argv) < 2: + return 2 + return _mode_aggregate_session(argv[1], argv[2:]) + if mode == "list-dirs": + return _mode_list_dirs() + if mode == "clean": + rest = argv[1:] + all_dirs = "--all-dirs" in rest or "-a" in rest + dry_run = "--dry-run" in rest or "-n" in rest + return _mode_clean(all_dirs=all_dirs, dry_run=dry_run) + sys.stderr.write(f"unknown mode: {mode}\n") + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/hooks/telemetry/clean-telemetry.sh b/.github/hooks/telemetry/clean-telemetry.sh new file mode 100755 index 000000000..5386ec711 --- /dev/null +++ b/.github/hooks/telemetry/clean-telemetry.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# +# clean-telemetry.sh +# Removes telemetry artifacts written by the Copilot telemetry hooks. By +# default it cleans this project's telemetry store; --all-dirs extends the +# cleanup to every registered project plus the user-level HVE home directory. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +readonly CORE_PY="${SCRIPT_DIR}/_telemetry_core.py" + +# Repo root anchors the default telemetry path so the script works from any cwd. +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel 2>/dev/null || true)" +[[ -n "${REPO_ROOT}" ]] || REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +readonly REPO_ROOT + +usage() { + cat <<'EOF' +Usage: clean-telemetry.sh [options] + +Removes telemetry artifacts (sessions-*.jsonl, raw-input.jsonl, .stacks/, +report.generated.html) from a telemetry store. Unrelated files are preserved. + +Options: + -a, --all-dirs Also clean every per-project telemetry directory recorded + in the user-level registry, plus the generated launchers, + report, and registry in the HVE home directory. + -p, --path DIR Telemetry directory. Default: /.copilot-tracking/telemetry + -n, --dry-run List what would be removed without deleting anything. + -y, --yes Skip the confirmation prompt (required for non-interactive + use). + -h, --help Show this help. + +Runs via: npm run telemetry:clean +EOF +} + +err() { + printf "ERROR: %s\n" "$1" >&2 + exit 1 +} + +# Prompt before destructive deletion. Aborts when no interactive terminal is +# available so cleanup never proceeds unattended without an explicit --yes. +confirm_cleanup() { + local scope_desc="$1" + printf "About to permanently remove telemetry artifacts from: %s\n" "${scope_desc}" + [[ -t 0 ]] || err "Confirmation required but no interactive terminal is available; re-run with --yes to proceed" + local reply + read -r -p "Continue? [y/N] " reply + case "${reply}" in + [yY]|[yY][eE][sS]) ;; + *) printf "Aborted.\n"; exit 0 ;; + esac +} + +main() { + local telemetry_path="${REPO_ROOT}/.copilot-tracking/telemetry" + local all_dirs=0 + local dry_run=0 + local assume_yes=0 + + while [[ $# -gt 0 ]]; do + case "$1" in + -a|--all-dirs) all_dirs=1; shift ;; + -p|--path) telemetry_path="$2"; shift 2 ;; + -n|--dry-run) dry_run=1; shift ;; + -y|--yes) assume_yes=1; shift ;; + -h|--help) usage; exit 0 ;; + *) err "Unknown option: $1" ;; + esac + done + + command -v python3 &>/dev/null || err "'python3' is required but not installed" + [[ -f "${CORE_PY}" ]] || err "Telemetry engine not found: ${CORE_PY}" + + if (( ! dry_run && ! assume_yes )); then + local scope_desc="${telemetry_path}" + (( all_dirs )) && scope_desc="ALL registered telemetry stores plus the user-level HVE home directory" + confirm_cleanup "${scope_desc}" + fi + + local -a args=("clean") + (( all_dirs )) && args+=("--all-dirs") + (( dry_run )) && args+=("--dry-run") + + HVE_TELEMETRY_DIR="${telemetry_path}" python3 "${CORE_PY}" "${args[@]}" +} + +main "$@" diff --git a/.github/hooks/telemetry/generate-telemetry-report.sh b/.github/hooks/telemetry/generate-telemetry-report.sh new file mode 100755 index 000000000..7e4f2edf0 --- /dev/null +++ b/.github/hooks/telemetry/generate-telemetry-report.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# +# generate-telemetry-report.sh +# Generates a self-contained telemetry report from hook JSONL files by +# embedding their contents into a copy of report.html. The generated report +# renders automatically without drag & drop. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +readonly TEMPLATE_PATH="${SCRIPT_DIR}/report.html" + +# Repo root anchors the default telemetry path so the script works from any cwd. +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel 2>/dev/null || true)" +[[ -n "${REPO_ROOT}" ]] || REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +readonly REPO_ROOT + +usage() { + cat <<'EOF' +Usage: generate-telemetry-report.sh [options] + +Options: + -d, --date DATE Target date (yyyy-MM-dd). Default: today (UTC). + Use 'all' to include every sessions-*.jsonl file. + -a, --all-dirs Scan every per-project telemetry directory recorded in + the user-level registry (~/.copilot/telemetry-dirs.txt) + for a combined cross-project report. + -p, --path DIR Telemetry directory. Default: /.copilot-tracking/telemetry + -l, --debug-log FILE Optional debug log JSONL (e.g. main.jsonl) for tokens. + When omitted, VS Code debug logs are auto-discovered and + the precise model version (e.g. claude-opus-4.6) plus + token data are joined in by session id. + -o, --output FILE Output path. Default: /report.generated.html + --open Open the generated report in the default browser. + -h, --help Show this help. + +Runs via: npm run telemetry:report +EOF +} + +err() { + printf "ERROR: %s\n" "$1" >&2 + exit 1 +} + +# Best-effort enrichment: extract llm_request events (precise model, token, and +# duration data) from VS Code debug logs, filtered to the session ids already +# collected from hook files. Writes matching events to $1. Returns non-zero when +# python3 is unavailable or no matching events are found (e.g. CLI-only users). +aggregate_debug_requests() { + local out="$1"; shift + command -v python3 &>/dev/null || return 1 + python3 "${SCRIPT_DIR}/_telemetry_core.py" aggregate-debug "$out" "$@" +} + +# Enrichment from CLI session state: reads ~/.copilot/session-state/{sid}/events.jsonl +# and produces llm_request-compatible events for the report. This provides model, +# token, and duration data for CLI sessions that have no VS Code debug logs. +aggregate_session_state() { + local out="$1"; shift + command -v python3 &>/dev/null || return 1 + python3 "${SCRIPT_DIR}/_telemetry_core.py" aggregate-session "$out" "$@" +} + +# Emit the user-level registry of per-project telemetry directories (one path +# per line), pruning stale entries. Used by --all-dirs for cross-project reports. +registry_dirs() { + command -v python3 &>/dev/null || return 0 + python3 "${SCRIPT_DIR}/_telemetry_core.py" list-dirs 2>/dev/null || true +} + +main() { + local target_date + target_date="$(date -u +%Y-%m-%d)" + local telemetry_path="${REPO_ROOT}/.copilot-tracking/telemetry" + local debug_log="" + local output_path="" + local open_report=0 + local all_dirs=0 + + # Temp files/dirs cleaned up on return (single trap to avoid overrides). + local -a tmp_files=() + # shellcheck disable=SC2154 # 't' is the loop variable, assigned within the trap body. + trap 'for t in "${tmp_files[@]:-}"; do [[ -n "$t" ]] && rm -rf "$t"; done' RETURN + + while [[ $# -gt 0 ]]; do + case "$1" in + -d|--date) target_date="$2"; shift 2 ;; + -a|--all-dirs) all_dirs=1; shift ;; + -p|--path) telemetry_path="$2"; shift 2 ;; + -l|--debug-log) debug_log="$2"; shift 2 ;; + -o|--output) output_path="$2"; shift 2 ;; + --open) open_report=1; shift ;; + -h|--help) usage; exit 0 ;; + *) err "Unknown option: $1" ;; + esac + done + + command -v jq &>/dev/null || err "'jq' is required but not installed" + [[ -f "${TEMPLATE_PATH}" ]] || err "Template not found: ${TEMPLATE_PATH}" + + # Default the output alongside the telemetry data it summarizes. + [[ -n "${output_path}" ]] || output_path="${telemetry_path}/report.generated.html" + + # Determine which telemetry directories to scan. With --all-dirs, prepend + # every directory recorded in the user-level registry (cross-project view). + declare -a search_dirs=() + if (( all_dirs )); then + while IFS= read -r d; do + [[ -n "$d" ]] && search_dirs+=("$d") + done < <(registry_dirs) + fi + search_dirs+=("${telemetry_path}") + + # Collect session files for the target date across the chosen directories, + # de-duplicating directories that appear more than once. + local pattern="sessions-${target_date}.jsonl" + [[ "${target_date}" == "all" ]] && pattern="sessions-*.jsonl" + declare -a files=() + declare -A seen_dirs=() + local dir + for dir in "${search_dirs[@]}"; do + [[ -n "$dir" && -z "${seen_dirs[$dir]:-}" ]] || continue + seen_dirs["$dir"]=1 + [[ -d "$dir" ]] || continue + while IFS= read -r -d '' f; do + files+=("$f") + done < <(find "$dir" -maxdepth 1 -name "${pattern}" -print0 | sort -z) + done + + if [[ -n "${debug_log}" ]]; then + [[ -f "${debug_log}" ]] || err "Debug log not found: ${debug_log}" + files+=("${debug_log}") + elif [[ ${#files[@]} -gt 0 ]]; then + # Auto-enrich with precise model + token data from VS Code debug logs, + # scoped to the sessions already collected. Silently skipped when none match. + local agg_dir agg_file + agg_dir="$(mktemp -d)" + tmp_files+=("${agg_dir}") + agg_file="${agg_dir}/debug-llm-requests.jsonl" + if aggregate_debug_requests "${agg_file}" "${files[@]}"; then + files+=("${agg_file}") + fi + + # Also enrich from CLI session state (provides model/token data for CLI users). + local cli_agg_file="${agg_dir}/cli-session-state.jsonl" + if aggregate_session_state "${cli_agg_file}" "${files[@]}"; then + files+=("${cli_agg_file}") + fi + fi + + if [[ ${#files[@]} -eq 0 ]]; then + printf "No telemetry files found in '%s' for date '%s'.\n" \ + "${telemetry_path}" "${target_date}" >&2 + exit 0 + fi + + # Build a compact JSON array of {name, content} objects with jq, then + # neutralize any literal so the embedded JSON cannot break out + # of its host ' ' + /id="embeddedData"/ { + printf "%s", tag_open + while ((getline line < data_file) > 0) printf "%s", line + close(data_file) + print tag_close + next + } + { print } + ' "${TEMPLATE_PATH}" > "${output_path}" + + printf "Wrote self-contained report: %s\n" "${output_path}" + printf "Embedded %d file(s): %s\n" "${#files[@]}" "$(IFS=', '; echo "${files[*]##*/}")" + + if [[ "${open_report}" -eq 1 ]]; then + if command -v xdg-open &>/dev/null; then + xdg-open "${output_path}" + elif command -v open &>/dev/null; then + open "${output_path}" + fi + fi +} + +main "$@" diff --git a/.github/hooks/telemetry/pyproject.toml b/.github/hooks/telemetry/pyproject.toml new file mode 100644 index 000000000..2601a4e3b --- /dev/null +++ b/.github/hooks/telemetry/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "hve-telemetry" +version = "0.0.0" +requires-python = ">=3.11" +dependencies = [] + +[dependency-groups] +dev = [ + "pytest>=9.0", + "ruff>=0.15", +] +# Atheris ships manylinux-only wheels; keep separate from dev so uv sync works on macOS. +fuzz = [ + "atheris>=3.0", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +python_files = ["test_*.py", "fuzz_harness.py"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] diff --git a/.github/hooks/telemetry/report.html b/.github/hooks/telemetry/report.html new file mode 100644 index 000000000..e95c4c86e --- /dev/null +++ b/.github/hooks/telemetry/report.html @@ -0,0 +1,808 @@ + + + + + + + +Local Generated Report + + + + +

Local Generated Report

+

Compare agents, models, instructions, and skills across local sessions

+ +
+

Drag & drop session JSONL files here or click to browse

+

sessions-*.jsonl · main.jsonl (debug logs for token data)

+ +
+
+ + + + +
+
+ +

Session Matrix

+
+ + + +
+ +

Loaded Instructions & Skills

+

Instruction and skill files a session opened via a tool call (estimated ~4 chars/token). Host-attached files are not counted here — see per-session Token Usage above for actual cost.

+
+ +

Tool Heatmap

+

Tool call counts per session (PreToolUse events)

+
+ +

Tool Latency

+

Avg tool execution time from PreToolUse→PostToolUse pairs (ms)

+
+
+ + + + diff --git a/.github/hooks/telemetry/telemetry-collector.sh b/.github/hooks/telemetry/telemetry-collector.sh new file mode 100755 index 000000000..c35abde60 --- /dev/null +++ b/.github/hooks/telemetry/telemetry-collector.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# +# telemetry-collector.sh +# Copilot hook handler that appends structured JSONL telemetry events. +# Uses Python3 for JSON processing. Fast no-op when telemetry is disabled. + +set -euo pipefail + +main() { + local input + input=$(cat) + + # Resolve repository root for reliable path anchoring across all surfaces + # (CLI, VS Code, cloud agent). Falls back to HVE_REPO_ROOT or cwd. + local repo_root + repo_root="$(git rev-parse --show-toplevel 2>/dev/null || true)" + [[ -z "$repo_root" ]] && repo_root="${HVE_REPO_ROOT:-.}" + + # Opt-in gate — exit immediately if telemetry is not enabled + if [[ "${HVE_TELEMETRY:-}" != "1" ]]; then + if [[ ! -f "$repo_root/.hve-telemetry" ]]; then + echo '{"continue":true}' + return 0 + fi + fi + + # Require Python3 for JSON processing + if ! command -v python3 &>/dev/null; then + echo "WARNING: HVE telemetry enabled but python3 not found — events will not be recorded" >&2 + echo '{"continue":true}' + return 0 + fi + + # Resolve the shared telemetry engine from the skill directory. + local script_dir core_py + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + core_py="${script_dir}/_telemetry_core.py" + + local telemetry_dir="${HVE_TELEMETRY_DIR:-$repo_root/.copilot-tracking/telemetry}" + mkdir -p "$telemetry_dir" "$telemetry_dir/.stacks" + + # Dump raw input for diagnostics (first 5 events only) + local raw_log="$telemetry_dir/raw-input.jsonl" + local raw_count=0 + if [[ -f "$raw_log" ]]; then + raw_count=$(wc -l < "$raw_log") + fi + if (( raw_count < 5 )); then + echo "$input" >> "$raw_log" + fi + + # Delegate all JSON processing to the shared telemetry engine. The engine + # records the event and, at Stop and PreCompact, enriches the session with + # token/cost data. + export HVE_REPO_ROOT="$repo_root" + export HVE_TELEMETRY_DIR="$telemetry_dir" + echo "$input" | python3 "$core_py" collect || true + + echo '{"continue":true}' +} + +main "$@" diff --git a/.github/hooks/telemetry/tests/fuzz_harness.py b/.github/hooks/telemetry/tests/fuzz_harness.py new file mode 100644 index 000000000..66299ef09 --- /dev/null +++ b/.github/hooks/telemetry/tests/fuzz_harness.py @@ -0,0 +1,129 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +"""Polyglot fuzz harness for telemetry core logic. + +Runs as a pytest test when Atheris is not installed. +Runs as an Atheris coverage-guided fuzz target when executed directly. +""" + +from __future__ import annotations + +import sys +from contextlib import suppress + +import _telemetry_core as core + +try: + import atheris +except ImportError: + atheris = None + FUZZING = False +else: + FUZZING = True + + +def fuzz_iter_jsonl(data: bytes) -> None: + """Fuzz JSONL parsing with arbitrary bytes.""" + provider = atheris.FuzzedDataProvider(data) + import tempfile + from pathlib import Path + + content = provider.ConsumeUnicodeNoSurrogates(500) + with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: + f.write(content) + f.flush() + list(core.iter_jsonl(f.name)) + Path(f.name).unlink(missing_ok=True) + + +def fuzz_normalize_event(data: bytes) -> None: + """Fuzz event normalization with arbitrary payloads.""" + provider = atheris.FuzzedDataProvider(data) + payload = {} + for key in ("hook_event_name", "hookEventName", "event"): + if provider.ConsumeBool(): + payload[key] = provider.ConsumeUnicodeNoSurrogates(30) + core._normalize_event(payload) + + +def fuzz_build_entry(data: bytes) -> None: + """Fuzz entry building with arbitrary payloads.""" + provider = atheris.FuzzedDataProvider(data) + events = [ + "SessionStart", "UserPromptSubmit", "PreToolUse", "PostToolUse", + "SubagentStart", "SubagentStop", "PreCompact", "Stop", "unknown", + ] + event = events[provider.ConsumeIntInRange(0, len(events) - 1)] + payload = { + "session_id": provider.ConsumeUnicodeNoSurrogates(20), + "timestamp": provider.ConsumeUnicodeNoSurrogates(30), + "tool_name": provider.ConsumeUnicodeNoSurrogates(15), + "prompt": provider.ConsumeUnicodeNoSurrogates(50), + } + import tempfile + from pathlib import Path + + stack_dir = Path(tempfile.mkdtemp()) + stack = core._AgentStack(stack_dir, payload["session_id"]) + with suppress(Exception): + core.build_entry(payload, event, stack) + # Cleanup + for f in stack_dir.iterdir(): + f.unlink(missing_ok=True) + stack_dir.rmdir() + + +FUZZ_TARGETS = [ + fuzz_iter_jsonl, + fuzz_normalize_event, + fuzz_build_entry, +] + + +def fuzz_dispatch(data: bytes) -> None: + """Route input to one fuzz target.""" + if len(data) < 2: + return + target_index = data[0] % len(FUZZ_TARGETS) + FUZZ_TARGETS[target_index](data[1:]) + + +class TestTelemetryFuzzHarness: + """Property tests mirroring fuzz-target behavior.""" + + def test_given_aliased_events_when_normalize_event_then_resolves(self) -> None: + assert core._normalize_event({"hook_event_name": "sessionStart"}) == "SessionStart" + assert core._normalize_event({"hook_event_name": "agentStop"}) == "Stop" + + def test_given_unknown_event_when_normalize_event_then_passes_through(self) -> None: + assert core._normalize_event({"hook_event_name": "CustomEvent"}) == "CustomEvent" + + def test_given_fallback_keys_when_normalize_event_then_resolves(self) -> None: + assert core._normalize_event({"hookEventName": "preToolUse"}) == "PreToolUse" + assert core._normalize_event({"event": "postToolUse"}) == "PostToolUse" + + def test_given_unknown_event_when_build_entry_then_returns_none(self, tmp_path) -> None: + stack = core._AgentStack(tmp_path / ".stacks", "sid") + assert core.build_entry({}, "unknown", stack) is None + + def test_given_session_start_when_build_entry_then_populates_fields(self, tmp_path) -> None: + stack = core._AgentStack(tmp_path / ".stacks", "sid") + entry = core.build_entry( + {"hook_event_name": "SessionStart", "source": "cli", "timestamp": "t"}, + "SessionStart", + stack, + ) + assert entry["event"] == "SessionStart" + assert entry["source"] == "cli" + + def test_given_non_dict_lines_when_iter_jsonl_then_skips_them(self, tmp_path) -> None: + f = tmp_path / "test.jsonl" + f.write_text('[1, 2]\n"string"\n{"valid": true}\n') + rows = list(core.iter_jsonl(f)) + assert rows == [{"valid": True}] + + +if __name__ == "__main__" and FUZZING: + atheris.instrument_all() + atheris.Setup(sys.argv, fuzz_dispatch) + atheris.Fuzz() diff --git a/.github/hooks/telemetry/tests/test_telemetry_core.py b/.github/hooks/telemetry/tests/test_telemetry_core.py new file mode 100644 index 000000000..1677971e5 --- /dev/null +++ b/.github/hooks/telemetry/tests/test_telemetry_core.py @@ -0,0 +1,687 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +"""Tests for the canonical telemetry engine (_telemetry_core).""" + +from __future__ import annotations + +import io +import json + +import _telemetry_core as core + + +def _write_jsonl(path, rows): + """Write a list of dicts as newline-delimited JSON.""" + path.write_text("".join(json.dumps(r) + "\n" for r in rows)) + + +def test_given_blank_and_malformed_lines_when_iter_jsonl_then_skips_them(tmp_path): + f = tmp_path / "events.jsonl" + f.write_text('{"a": 1}\n\n \nnot-json\n{"b": 2}\n') + rows = list(core.iter_jsonl(f)) + assert rows == [{"a": 1}, {"b": 2}] + + +def test_given_missing_file_when_iter_jsonl_then_returns_empty(tmp_path): + assert list(core.iter_jsonl(tmp_path / "nope.jsonl")) == [] + + +def test_given_overlapping_sids_when_collect_sids_then_dedups(tmp_path): + a = tmp_path / "a.jsonl" + b = tmp_path / "b.jsonl" + _write_jsonl(a, [{"sid": "s1"}, {"sid": "s2"}, {"event": "x"}]) + _write_jsonl(b, [{"sid": "s2"}, {"sid": "s3"}]) + assert core.collect_sids([str(a), str(b)]) == {"s1", "s2", "s3"} + + +def test_given_lock_pid_when_find_process_log_then_resolves_path(tmp_path): + home = tmp_path / "home" + state_dir = home / "session-state" / "sid1" + state_dir.mkdir(parents=True) + (state_dir / "inuse.4242.lock").write_text("") + logs = home / "logs" + logs.mkdir() + target = logs / "process-abc-4242.log" + target.write_text("{}\n") + assert core.find_process_log(state_dir, home) == str(target) + + +def test_given_no_lock_when_find_process_log_then_returns_none(tmp_path): + home = tmp_path / "home" + state_dir = home / "session-state" / "sid1" + state_dir.mkdir(parents=True) + assert core.find_process_log(state_dir, home) is None + + +def test_given_mixed_blocks_when_parse_process_log_then_filters_by_interaction_and_kind(tmp_path): + log = tmp_path / "process.log" + log.write_text( + "{\n" + ' "kind": "assistant_usage",\n' + ' "properties": {"interaction_id": "i1", "model": "m"},\n' + ' "metrics": {"output_tokens": 5}\n' + "}\n" + "{\n" + ' "kind": "assistant_usage",\n' + ' "properties": {"interaction_id": "other"}\n' + "}\n" + "{\n" + ' "kind": "something_else"\n' + "}\n" + ) + entries = core.parse_process_log(str(log), {"i1"}) + assert len(entries) == 1 + assert entries[0]["properties"]["interaction_id"] == "i1" + + +def test_given_session_events_when_scan_session_state_then_collects_metadata(tmp_path): + state = tmp_path / "events.jsonl" + _write_jsonl( + state, + [ + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:01Z", + "data": {"model": "gpt", "interactionId": "i1"}, + }, + { + "type": "assistant.turn_start", + "timestamp": "2026-01-01T00:00:00Z", + "data": {"interactionId": "i2"}, + }, + { + "type": "session.model_change", + "timestamp": "2026-01-01T00:00:02Z", + "data": {"reasoningEffort": "high"}, + }, + { + "type": "subagent.started", + "timestamp": "2026-01-01T00:00:03Z", + "data": {"toolCallId": "t1", "agentName": "Researcher"}, + }, + ], + ) + meta = core.scan_session_state(state) + assert meta["messages"] == 1 + assert meta["turns"] == 1 + assert meta["models"] == {"gpt": 1} + assert meta["interaction_ids"] == {"i1", "i2"} + assert meta["reasoning_effort"] == "high" + assert meta["subagent_map"] == {"t1": "Researcher"} + assert meta["first_ts"] == "2026-01-01T00:00:00Z" + assert meta["last_ts"] == "2026-01-01T00:00:03Z" + + +def _make_session(tmp_path, sid, state_rows, process_rows=None, pid=None, write_lock=True): + """Create a minimal session directory tree with optional process log. + + Set ``write_lock=False`` to simulate a session that has ended (its lock + file removed) while its process log still exists, exercising the + interaction-id fallback path. + """ + home = tmp_path / "home" + state_dir = home / "session-state" / sid + state_dir.mkdir(parents=True) + _write_jsonl(state_dir / "events.jsonl", state_rows) + if pid is not None and write_lock: + (state_dir / f"inuse.{pid}.lock").write_text("") + if process_rows is not None and pid is not None: + logs = home / "logs" + logs.mkdir(exist_ok=True) + # Build process-log blocks in the brace-delimited format the parser + # expects (top-level '{' on its own line, not JSONL). + blocks = [] + for r in process_rows: + inner = json.dumps(r, indent=2) + blocks.append(inner + "\n") + text = "".join(blocks) + (logs / f"process-x-{pid}.log").write_text(text) + return home, state_dir, state_dir / "events.jsonl" + + +def test_given_process_log_when_build_session_summary_then_uses_process_log(tmp_path): + state_rows = [ + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:00Z", + "data": {"model": "m", "interactionId": "i1"}, + } + ] + process_rows = [ + { + "kind": "assistant_usage", + "properties": {"interaction_id": "i1", "model": "m"}, + "metrics": { + "input_tokens": 10, + "input_tokens_uncached": 7, + "output_tokens": 20, + "cache_read_tokens": 1, + "cache_write_tokens": 2, + "total_nano_aiu": 99, + }, + } + ] + home, state_dir, state_file = _make_session( + tmp_path, "sid1", state_rows, process_rows, pid=777 + ) + summary = core.build_session_summary("sid1", state_dir, state_file, home) + assert summary["input_tokens"] == 10 + assert summary["input_tokens_uncached"] == 7 + assert summary["output_tokens"] == 20 + assert summary["cache_write_tokens"] == 2 + assert summary["total_nano_aiu"] == 99 + assert summary["model_usage"]["m"]["input_tokens"] == 10 + assert summary["model_usage"]["m"]["input_tokens_uncached"] == 7 + assert summary["token_source"] == "process_log" + + +def test_given_ended_session_when_build_summary_then_matches_log_by_iid(tmp_path): + state_rows = [ + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:00Z", + "data": {"model": "m", "interactionId": "i1"}, + } + ] + process_rows = [ + { + "kind": "assistant_usage", + "properties": {"interaction_id": "i1", "model": "m"}, + "metrics": { + "input_tokens": 30, + "input_tokens_uncached": 5, + "output_tokens": 12, + "cache_read_tokens": 25, + "cache_write_tokens": 0, + "total_nano_aiu": 42, + }, + } + ] + # pid names the process log file, but write_lock=False removes the lock so + # the PID-based lookup fails and the interaction-id scan must recover it. + home, state_dir, state_file = _make_session( + tmp_path, "sid1", state_rows, process_rows, pid=888, write_lock=False + ) + summary = core.build_session_summary("sid1", state_dir, state_file, home) + assert summary["token_source"] == "process_log" + assert summary["input_tokens"] == 30 + assert summary["input_tokens_uncached"] == 5 + + +def test_given_no_process_log_when_build_session_summary_then_falls_back_to_state(tmp_path): + state_rows = [ + { + "type": "session.shutdown", + "timestamp": "2026-01-01T00:00:01Z", + "data": { + "modelMetrics": { + "m": { + "requests": {"count": 2}, + "usage": { + "inputTokens": 12, + "outputTokens": 7, + "cacheReadTokens": 4, + "cacheWriteTokens": 5, + }, + "totalNanoAiu": 50, + } + } + }, + }, + ] + home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) + summary = core.build_session_summary("sid1", state_dir, state_file, home) + assert summary["output_tokens"] == 7 + assert summary["input_tokens"] == 12 + assert summary["cache_read_tokens"] == 4 + # Unified schema always reports cache_write_tokens. + assert summary["cache_write_tokens"] == 5 + assert summary["token_source"] == "state_fallback" + assert summary["total_nano_aiu"] == 50 + # inputTokens includes cache, so fresh input is recovered by subtraction. + assert summary["input_tokens_uncached"] == 3 + + +def test_given_resumed_session_when_build_session_summary_then_sums_shutdowns(tmp_path): + def _shutdown(ts, in_tok, out_tok, cr, cw, nano): + return { + "type": "session.shutdown", + "timestamp": ts, + "data": { + "modelMetrics": { + "m": { + "requests": {"count": 1}, + "usage": { + "inputTokens": in_tok, + "outputTokens": out_tok, + "cacheReadTokens": cr, + "cacheWriteTokens": cw, + }, + "totalNanoAiu": nano, + } + } + }, + } + + state_rows = [ + _shutdown("2026-01-01T00:00:01Z", 10, 3, 4, 2, 20), + _shutdown("2026-01-01T00:00:02Z", 30, 5, 8, 6, 40), + ] + home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) + summary = core.build_session_summary("sid1", state_dir, state_file, home) + assert summary["token_source"] == "state_fallback" + assert summary["input_tokens"] == 40 + assert summary["output_tokens"] == 8 + assert summary["cache_read_tokens"] == 12 + assert summary["cache_write_tokens"] == 8 + assert summary["total_nano_aiu"] == 60 + # Fresh input summed per segment: (10-4-2) + (30-8-6) = 4 + 16 = 20. + assert summary["input_tokens_uncached"] == 20 + + +def test_given_no_shutdown_when_build_session_summary_then_input_unknown(tmp_path): + state_rows = [ + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:00Z", + "data": {"model": "m", "outputTokens": 7, "interactionId": "i1"}, + }, + ] + home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) + summary = core.build_session_summary("sid1", state_dir, state_file, home) + assert summary["output_tokens"] == 7 + # No shutdown segment exists, so input is unknown (None), not a true zero. + assert summary["input_tokens"] is None + assert summary["cache_read_tokens"] is None + assert summary["total_nano_aiu"] is None + assert summary["token_source"] == "state_fallback" + # Fresh input is unknown, so the key is omitted. + assert "input_tokens_uncached" not in summary + + +def test_given_shutdown_missing_metrics_when_summary_then_output_from_messages(tmp_path): + # One segment ends with modelMetrics; a later segment aborts without them. + # assistant.message output stays complete, so the message sum (3+9=12) + # must win over the lone shutdown's output (3). + state_rows = [ + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:00Z", + "data": {"model": "m", "outputTokens": 3, "interactionId": "i1"}, + }, + { + "type": "session.shutdown", + "timestamp": "2026-01-01T00:00:01Z", + "data": { + "modelMetrics": { + "m": { + "requests": {"count": 1}, + "usage": { + "inputTokens": 10, + "outputTokens": 3, + "cacheReadTokens": 4, + "cacheWriteTokens": 2, + }, + "totalNanoAiu": 20, + } + } + }, + }, + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:02Z", + "data": {"model": "m", "outputTokens": 9, "interactionId": "i2"}, + }, + # Aborted resume segment: shutdown present but no modelMetrics. + { + "type": "session.shutdown", + "timestamp": "2026-01-01T00:00:03Z", + "data": {}, + }, + ] + home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) + summary = core.build_session_summary("sid1", state_dir, state_file, home) + assert summary["token_source"] == "state_fallback" + # Output reconciled to the complete message sum, not the undercounting shutdown. + assert summary["output_tokens"] == 12 + assert summary["model_usage"]["m"]["output_tokens"] == 12 + # Input/cache/AIU still come from the one segment that reported metrics. + assert summary["input_tokens"] == 10 + assert summary["cache_read_tokens"] == 4 + assert summary["input_tokens_uncached"] == 4 + + +def test_given_single_element_stack_when_current_then_returns_full_name(tmp_path): + stack = core._AgentStack(tmp_path / ".stacks", "sid1") + stack.push("Researcher") + assert stack.current() == "Researcher" + + +def test_given_subagent_pushed_when_build_entry_pretooluse_then_reports_agent(tmp_path): + stack = core._AgentStack(tmp_path / ".stacks", "sid1") + core.build_entry( + {"hook_event_name": "SubagentStart", "agent_name": "Coder"}, "SubagentStart", stack + ) + entry = core.build_entry( + {"hook_event_name": "PreToolUse", "tool_name": "read"}, "PreToolUse", stack + ) + assert entry["agent"] == "Coder" + + +def test_given_unknown_event_when_build_entry_then_returns_none(tmp_path): + stack = core._AgentStack(tmp_path / ".stacks", "sid1") + assert core.build_entry({"hook_event_name": "unknown"}, "unknown", stack) is None + + +def test_given_skill_path_when_build_entry_then_detects_skill(tmp_path): + stack = core._AgentStack(tmp_path / ".stacks", "sid1") + skill_file = tmp_path / "SKILL.md" + skill_file.write_text("x" * 40) + data = { + "hook_event_name": "PreToolUse", + "tool_name": "read", + "tool_input": { + "filePath": "/repo/.github/skills/coll/my-skill/SKILL.md" + }, + } + entry = core.build_entry(data, "PreToolUse", stack) + assert entry["skill"] == "my-skill" + + +def test_given_stop_event_when_mode_collect_then_writes_entry_and_summary(tmp_path, monkeypatch): + tel_dir = tmp_path / "tel" + home = tmp_path / "home" + state_dir = home / "session-state" / "sid1" + state_dir.mkdir(parents=True) + _write_jsonl( + state_dir / "events.jsonl", + [ + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:00Z", + "data": {"model": "m", "outputTokens": 9}, + } + ], + ) + monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) + monkeypatch.setenv("COPILOT_HOME", str(home)) + payload = { + "hook_event_name": "Stop", + "session_id": "sid1", + "timestamp": "2026-01-02T00:00:00Z", + } + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) + assert core._mode_collect() == 0 + + logs = list(tel_dir.glob("sessions-*.jsonl")) + assert len(logs) == 1 + events = [json.loads(line) for line in logs[0].read_text().splitlines()] + assert events[0]["event"] == "Stop" + assert any(e["event"] == "SessionSummary" for e in events) + + +def test_given_precompact_event_when_mode_collect_then_writes_summary(tmp_path, monkeypatch): + tel_dir = tmp_path / "tel" + home = tmp_path / "home" + state_dir = home / "session-state" / "sid1" + state_dir.mkdir(parents=True) + _write_jsonl( + state_dir / "events.jsonl", + [ + { + "type": "assistant.message", + "timestamp": "2026-01-01T00:00:00Z", + "data": {"model": "m", "outputTokens": 9}, + } + ], + ) + monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) + monkeypatch.setenv("COPILOT_HOME", str(home)) + payload = { + "hook_event_name": "PreCompact", + "session_id": "sid1", + "timestamp": "2026-01-02T00:00:00Z", + } + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) + assert core._mode_collect() == 0 + + logs = list(tel_dir.glob("sessions-*.jsonl")) + assert len(logs) == 1 + events = [json.loads(line) for line in logs[0].read_text().splitlines()] + assert events[0]["event"] == "PreCompact" + # PreCompact captures a summary before process logs rotate. + assert any(e["event"] == "SessionSummary" for e in events) + + +def test_given_traversal_sid_when_mode_collect_then_rejects(tmp_path, monkeypatch): + tel_dir = tmp_path / "tel" + monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) + payload = {"hook_event_name": "SessionStart", "session_id": "../escape"} + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) + assert core._mode_collect() == 0 + # No telemetry written for a rejected sid. + assert not tel_dir.exists() or not list(tel_dir.glob("sessions-*.jsonl")) + + +def test_given_new_dir_when_register_telemetry_dir_then_appends_absolute_path(tmp_path): + registry = tmp_path / "telemetry-dirs.txt" + tel_dir = tmp_path / "proj" / "tel" + tel_dir.mkdir(parents=True) + core.register_telemetry_dir(tel_dir, registry) + assert core.read_registry_dirs(registry) == [str(tel_dir.resolve())] + + +def test_given_existing_entry_when_register_telemetry_dir_then_dedups(tmp_path): + registry = tmp_path / "telemetry-dirs.txt" + tel_dir = tmp_path / "tel" + tel_dir.mkdir() + core.register_telemetry_dir(tel_dir, registry) + core.register_telemetry_dir(tel_dir, registry) + assert core.read_registry_dirs(registry) == [str(tel_dir.resolve())] + + +def test_given_blank_and_duplicate_lines_when_read_registry_dirs_then_normalizes(tmp_path): + registry = tmp_path / "telemetry-dirs.txt" + registry.write_text("/a\n\n \n/b\n/a\n", encoding="utf-8") + assert core.read_registry_dirs(registry) == ["/a", "/b"] + + +def test_given_session_start_when_mode_collect_then_registers_dir(tmp_path, monkeypatch): + tel_dir = tmp_path / "tel" + home = tmp_path / "home" + hve = tmp_path / "hve" + monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) + monkeypatch.setenv("COPILOT_HOME", str(home)) + monkeypatch.setenv("HVE_HOME", str(hve)) + payload = {"hook_event_name": "SessionStart", "session_id": "sid1"} + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) + assert core._mode_collect() == 0 + assert core.read_registry_dirs(hve / "telemetry-dirs.txt") == [str(tel_dir.resolve())] + # A cross-project launcher for the host platform is emitted in the HVE home. + if core._is_windows(): + assert (hve / "generate-report.ps1").is_file() + assert (hve / "clean-telemetry.ps1").is_file() + else: + assert (hve / "generate-report.sh").is_file() + assert (hve / "clean-telemetry.sh").is_file() + + +def test_given_stale_entries_when_mode_list_dirs_then_prunes_and_prints( + tmp_path, monkeypatch, capsys +): + hve = tmp_path / "hve" + hve.mkdir() + live = tmp_path / "live" + live.mkdir() + dead = tmp_path / "dead" + registry = hve / "telemetry-dirs.txt" + registry.write_text(f"{live}\n{dead}\n", encoding="utf-8") + monkeypatch.setenv("HVE_HOME", str(hve)) + assert core._mode_list_dirs() == 0 + out = capsys.readouterr().out.splitlines() + assert out == [str(live)] + # Dead entry is pruned from the registry on read. + assert core.read_registry_dirs(registry) == [str(live)] + + +def test_given_posix_when_write_report_launchers_then_writes_sh_only(tmp_path, monkeypatch): + hve = tmp_path / "hve" + script_dir = tmp_path / "hook" + script_dir.mkdir() + monkeypatch.setenv("HVE_HOME", str(hve)) + monkeypatch.setattr(core, "_is_windows", lambda: False) + core.write_report_launchers(script_dir) + report_script = str(script_dir / "generate-telemetry-report.sh") + out_path = str(hve / "report.generated.html") + sh = (hve / "generate-report.sh").read_text(encoding="utf-8") + assert report_script in sh + assert out_path in sh + assert "--all-dirs" in sh + # No PowerShell launcher on POSIX. + assert not (hve / "generate-report.ps1").exists() + + +def test_given_windows_when_write_report_launchers_then_writes_ps1_only(tmp_path, monkeypatch): + hve = tmp_path / "hve" + script_dir = tmp_path / "hook" + script_dir.mkdir() + monkeypatch.setenv("HVE_HOME", str(hve)) + monkeypatch.setattr(core, "_is_windows", lambda: True) + core.write_report_launchers(script_dir) + report_script = str(script_dir / "generate-telemetry-report.sh") + out_path = str(hve / "report.generated.html") + ps = (hve / "generate-report.ps1").read_text(encoding="utf-8") + assert report_script in ps + assert out_path in ps + assert "--all-dirs" in ps + # No POSIX launcher on Windows. + assert not (hve / "generate-report.sh").exists() + + +def test_given_posix_when_write_report_launchers_then_clean_sh_delegates_to_bash( + tmp_path, monkeypatch +): + hve = tmp_path / "hve" + script_dir = tmp_path / "hook" + script_dir.mkdir() + monkeypatch.setenv("HVE_HOME", str(hve)) + monkeypatch.setattr(core, "_is_windows", lambda: False) + core.write_report_launchers(script_dir) + clean_script = str(script_dir / "clean-telemetry.sh") + sh = (hve / "clean-telemetry.sh").read_text(encoding="utf-8") + assert clean_script in sh + assert "--all-dirs" in sh + assert not (hve / "clean-telemetry.ps1").exists() + + +def test_given_windows_when_write_report_launchers_then_clean_ps1_is_native( + tmp_path, monkeypatch +): + hve = tmp_path / "hve" + script_dir = tmp_path / "hook" + script_dir.mkdir() + monkeypatch.setenv("HVE_HOME", str(hve)) + monkeypatch.setattr(core, "_is_windows", lambda: True) + core.write_report_launchers(script_dir) + clean_ps1 = str(script_dir / "Invoke-TelemetryClean.ps1") + ps = (hve / "clean-telemetry.ps1").read_text(encoding="utf-8") + # Native delegation to the PowerShell wrapper, no bash. + assert clean_ps1 in ps + assert "-AllDirs" in ps + assert "bash" not in ps + assert not (hve / "clean-telemetry.sh").exists() + + +def _seed_telemetry_store(tel_dir): + """Populate a telemetry directory with representative artifacts plus an + unrelated file that cleanup must preserve.""" + tel_dir.mkdir(parents=True) + (tel_dir / "sessions-2026-01-01.jsonl").write_text("{}\n", encoding="utf-8") + (tel_dir / "sessions-2026-01-02.jsonl").write_text("{}\n", encoding="utf-8") + (tel_dir / "raw-input.jsonl").write_text("{}\n", encoding="utf-8") + (tel_dir / "report.generated.html").write_text("", encoding="utf-8") + stacks = tel_dir / ".stacks" + stacks.mkdir() + (stacks / "sid1.json").write_text("[]", encoding="utf-8") + keep = tel_dir / "keep-me.txt" + keep.write_text("user data", encoding="utf-8") + return keep + + +def test_given_store_when_clean_telemetry_dir_then_removes_only_artifacts(tmp_path): + tel_dir = tmp_path / "tel" + keep = _seed_telemetry_store(tel_dir) + removed = [] + core.clean_telemetry_dir(tel_dir, dry_run=False, removed=removed) + assert not (tel_dir / "sessions-2026-01-01.jsonl").exists() + assert not (tel_dir / "sessions-2026-01-02.jsonl").exists() + assert not (tel_dir / "raw-input.jsonl").exists() + assert not (tel_dir / "report.generated.html").exists() + assert not (tel_dir / ".stacks").exists() + # Unrelated files are preserved. + assert keep.exists() + assert len(removed) == 5 + + +def test_given_dry_run_when_clean_telemetry_dir_then_reports_without_deleting(tmp_path): + tel_dir = tmp_path / "tel" + _seed_telemetry_store(tel_dir) + removed = [] + core.clean_telemetry_dir(tel_dir, dry_run=True, removed=removed) + assert (tel_dir / "raw-input.jsonl").exists() + assert (tel_dir / ".stacks").exists() + assert len(removed) == 5 + + +def test_given_current_store_when_mode_clean_then_cleans_only_current( + tmp_path, monkeypatch +): + current = tmp_path / "current" + other = tmp_path / "other" + hve = tmp_path / "hve" + _seed_telemetry_store(current) + _seed_telemetry_store(other) + hve.mkdir() + registry = hve / "telemetry-dirs.txt" + registry.write_text(f"{other}\n", encoding="utf-8") + monkeypatch.setenv("HVE_TELEMETRY_DIR", str(current)) + monkeypatch.setenv("HVE_HOME", str(hve)) + assert core._mode_clean(all_dirs=False, dry_run=False) == 0 + assert not (current / "raw-input.jsonl").exists() + # Without --all-dirs, registered stores and the registry are untouched. + assert (other / "raw-input.jsonl").exists() + assert registry.exists() + + +def test_given_all_dirs_when_mode_clean_then_cleans_registry_and_home( + tmp_path, monkeypatch +): + current = tmp_path / "current" + other = tmp_path / "other" + hve = tmp_path / "hve" + _seed_telemetry_store(current) + _seed_telemetry_store(other) + hve.mkdir() + registry = hve / "telemetry-dirs.txt" + registry.write_text(f"{other}\n", encoding="utf-8") + (hve / "report.generated.html").write_text("", encoding="utf-8") + (hve / "generate-report.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + monkeypatch.setenv("HVE_TELEMETRY_DIR", str(current)) + monkeypatch.setenv("HVE_HOME", str(hve)) + assert core._mode_clean(all_dirs=True, dry_run=False) == 0 + assert not (current / "raw-input.jsonl").exists() + assert not (other / "raw-input.jsonl").exists() + assert not registry.exists() + assert not (hve / "report.generated.html").exists() + assert not (hve / "generate-report.sh").exists() + + +def test_given_clean_mode_when_main_dispatches_then_parses_flags(tmp_path, monkeypatch): + current = tmp_path / "current" + _seed_telemetry_store(current) + monkeypatch.setenv("HVE_TELEMETRY_DIR", str(current)) + assert core.main(["clean", "--dry-run"]) == 0 + # Dry-run leaves artifacts in place. + assert (current / "raw-input.jsonl").exists() + + diff --git a/.github/hooks/telemetry/uv.lock b/.github/hooks/telemetry/uv.lock new file mode 100644 index 000000000..deac25054 --- /dev/null +++ b/.github/hooks/telemetry/uv.lock @@ -0,0 +1,123 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "atheris" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/58/5965955898e16bee17c8379eae12194993bf641c4629016991248b862069/atheris-3.0.0.tar.gz", hash = "sha256:1f0929c7bc3040f3fe4102e557718734190cf2d7718bbb8e3ce6d3eb56ef5bb3", size = 373239, upload-time = "2025-11-24T23:54:02.15Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/15/cf109e2e8696a54c8c4bc3ef79a79bec32361eceb64eaa36690a682e83a9/atheris-3.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8a5c8a781467c187da40fd29139784193e2647058831f837f675d0bb8cbd8746", size = 34805555, upload-time = "2025-11-24T23:53:53.477Z" }, + { url = "https://files.pythonhosted.org/packages/85/8c/e9960b996e70e5f6a523670431166b2b238de52fef094955515dcf854da1/atheris-3.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:510e502c57b6dc615fb174066407af620d4c7f73cf08a782c86e7761bf12c4eb", size = 34907016, upload-time = "2025-11-24T23:53:56.535Z" }, + { url = "https://files.pythonhosted.org/packages/db/48/df670f75f458cc7c1752a01a394fd59c830b08172dd59cf29d73f31050f9/atheris-3.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac", size = 34858525, upload-time = "2025-11-24T23:53:59.925Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "hve-telemetry" +version = "0.0.0" +source = { virtual = "." } + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] +fuzz = [ + { name = "atheris" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.0" }, + { name = "ruff", specifier = ">=0.15" }, +] +fuzz = [{ name = "atheris", specifier = ">=3.0" }] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/5f7ec371001337d8fa61701c186ff8b613ecac1651848c5950f4c4d5f2e9/ruff-0.15.16.tar.gz", hash = "sha256:d05e78d38c78caf020b03789e25106c93017db5a0cb6e2819885018c61343b78", size = 4714267, upload-time = "2026-06-04T16:33:09.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/42/53ef1c3953f157956db9bf7861e3bc50b9b887ce93300aa48cdba8336fe6/ruff-0.15.16-py3-none-linux_armv6l.whl", hash = "sha256:6ac3c0b3969cc6cf6b158c4e2f8f682acb58e7d700d8a44b65ecdc72d66ab0b2", size = 10709025, upload-time = "2026-06-04T16:32:51.935Z" }, + { url = "https://files.pythonhosted.org/packages/93/9a/a79159346f19134a956607754e57d8d128f7a4c00f4ad2f7514d224c172c/ruff-0.15.16-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:197c207ed75ffba54a0dec23db4aa939a27a3053073e085e0042433cbdc58e4a", size = 11063550, upload-time = "2026-06-04T16:32:42.24Z" }, + { url = "https://files.pythonhosted.org/packages/bc/72/3ce2ac000a5299ec238e01f51397b3b653c93b077d9b1bfe8715bb895f20/ruff-0.15.16-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3a39fec45ab316cc23e7558f23fea4a70403ddb5648ea9a4a3854a16973d0071", size = 10421345, upload-time = "2026-06-04T16:32:37.251Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c2/cc7fad3ec9169373f5b6a18f1917b91080feec40c3f9658334a1d28e2f03/ruff-0.15.16-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba93191d79003116b95128c9d306e045200fdbd0bccb782b110f3cd1d4abc5cf", size = 10757217, upload-time = "2026-06-04T16:32:54.722Z" }, + { url = "https://files.pythonhosted.org/packages/69/d2/3474009eaa0a65b31fa7152a2fad5e2f050c640ceb1e6b02ee6922e94c82/ruff-0.15.16-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6ee4b90520630120ef032aa5cc10db483852dff950e78b1d717e2993a61ac8d", size = 10507035, upload-time = "2026-06-04T16:33:05.343Z" }, + { url = "https://files.pythonhosted.org/packages/ca/81/b7ae6ccbd11f0c8dc3d5d67fc4be9b57ff57ca86ba56152021378e1277f2/ruff-0.15.16-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e4215bc938bc3c8215c1472c1aa437e310fee20cd427335fec9d7e609563628", size = 11255291, upload-time = "2026-06-04T16:32:49.49Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e1/46e526f1a7cc90857ce6ddf25fbb77eb6568651ac38d71b033af07076dd5/ruff-0.15.16-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c8d26be963b090f10e29abc8b3e74a2a321f6fa34e02424e30b5af89350ecbb", size = 12124922, upload-time = "2026-06-04T16:33:07.821Z" }, + { url = "https://files.pythonhosted.org/packages/1a/da/5c791b088b596b24d0deb967fa28ae02ad751a140c0b9ea81c5ab915d6c0/ruff-0.15.16-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f198cf4123602a2280ed46c307bcbafe41758d6fee5b456b6b6058ca1514b3b4", size = 11332186, upload-time = "2026-06-04T16:33:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/72/11/5da87abe20047c8962361473923ebb2f62b595250126aadfad8c20649c1e/ruff-0.15.16-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb27515fa6240fb586ae82b901a59e67d24acff86f2190b433dc542fe0435aeb", size = 11373541, upload-time = "2026-06-04T16:32:47.007Z" }, + { url = "https://files.pythonhosted.org/packages/fe/2a/8554754c23a854ae3fd6b507e36ad61ddb121e298c6d5d617dec94ed0f14/ruff-0.15.16-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a267c46ba1593fc26b8eecbea050b39d40c0b6bb7781ee11c90a02cd10032951", size = 11353014, upload-time = "2026-06-04T16:32:34.795Z" }, + { url = "https://files.pythonhosted.org/packages/62/25/62ea41529ec89f742ea3fed9cb1059c72877ec7cf9b9e99ac9cf3294d1d9/ruff-0.15.16-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:528c68f39a91498a8d50e91ff5985df3d105782bab49cc378e73ac26bff083e8", size = 10737467, upload-time = "2026-06-04T16:32:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/90/17/334d3ad9de4d40f9dd58fdd09e35ce64553bb501e2f19a839e2fb6be14fc/ruff-0.15.16-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7ed55c58950df60589a9a7a5d2f8fa5f54ebd287163be805adfe6ee95a9de123", size = 10521910, upload-time = "2026-06-04T16:32:32.54Z" }, + { url = "https://files.pythonhosted.org/packages/4d/bd/3ac7c6ae77a885c1004b3dda2446ea401768d24f851c14b4ad4b24f6639c/ruff-0.15.16-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d482feaf51512b50f9790ceb417a56a61dd1e9d9bf967662b9ed27c01b34f53a", size = 10979190, upload-time = "2026-06-04T16:32:57.492Z" }, + { url = "https://files.pythonhosted.org/packages/33/d7/609546e6a413c3f216fbf2a50c928f97c80939154f6a0503114094a86191/ruff-0.15.16-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e15bc8c94513dae2a40cc9ef07c94fdd4ecc9e29dabebeebe170f952322c9e3", size = 11477014, upload-time = "2026-06-04T16:32:44.687Z" }, + { url = "https://files.pythonhosted.org/packages/74/0d/f2cd247ad32633a5c36e97141a2c21b11c6279f7957bc2ff360b1e08fddd/ruff-0.15.16-py3-none-win32.whl", hash = "sha256:580378f7bd4aa25f72e74aa54948a9622f142b1e509521dd10902e886681cc1e", size = 10735541, upload-time = "2026-06-04T16:32:30.145Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/02e845ef151b1dee585e55c4739f8e1734ae1d9f1221dff65761c162208b/ruff-0.15.16-py3-none-win_amd64.whl", hash = "sha256:408256017284eddf98fff77b29aa4fb30f586042d535b2d9befc6512f400aaec", size = 11843403, upload-time = "2026-06-04T16:32:39.76Z" }, + { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, +] diff --git a/.gitignore b/.gitignore index 411bd95d0..66d59a7f0 100644 --- a/.gitignore +++ b/.gitignore @@ -480,6 +480,9 @@ dependency-pinning-artifacts/ # Copilot tracking .copilot-tracking/ +# Local telemetry opt-in marker +.hve-telemetry + # MCP server configuration - exclude local token files .mcp/*-local.json .mcp/*.local.json diff --git a/collections/hve-core-all.collection.md b/collections/hve-core-all.collection.md index 53613f55e..7460d24e9 100644 --- a/collections/hve-core-all.collection.md +++ b/collections/hve-core-all.collection.md @@ -279,4 +279,10 @@ Use this edition when you want access to everything without choosing a focused c | **video-to-gif** | Video-to-GIF conversion with FFmpeg two-pass optimization | | **vscode-playwright** | VS Code screenshot capture using Playwright MCP with serve-web for slide decks and documentation | +### Hooks + +| Name | Description | +|---------------|----------------------------------------------------------------------------| +| **telemetry** | Records Copilot session lifecycle events to local telemetry for reporting. | + diff --git a/collections/hve-core-all.collection.yml b/collections/hve-core-all.collection.yml index 4f5cc511d..5056b3f89 100644 --- a/collections/hve-core-all.collection.yml +++ b/collections/hve-core-all.collection.yml @@ -599,6 +599,8 @@ items: kind: skill - path: .github/skills/shared/telemetry-foundations kind: skill +- path: .github/hooks/telemetry.json + kind: hook display: featured: true ordering: alpha diff --git a/collections/hve-core.collection.md b/collections/hve-core.collection.md index 4905ace4c..881fcb66c 100644 --- a/collections/hve-core.collection.md +++ b/collections/hve-core.collection.md @@ -83,4 +83,10 @@ HVE Core provides the flagship RPI (Research, Plan, Implement, Review) workflow | **telemetry-foundations** | Declarative OpenTelemetry-aligned telemetry vocabulary and instrumentation conventions for traces, metrics, logs, and PII handling | | **vally-tests** | Authors Vally conformance tests for prompts, instructions, agents, and skills, including refusals for jailbreak, prompt-injection, harmful-elicitation, TOS, CoC, and PII-extraction stimuli | +### Hooks + +| Name | Description | +|---------------|----------------------------------------------------------------------------| +| **telemetry** | Records Copilot session lifecycle events to local telemetry for reporting. | + diff --git a/collections/hve-core.collection.yml b/collections/hve-core.collection.yml index e002320e5..4b7114e22 100644 --- a/collections/hve-core.collection.yml +++ b/collections/hve-core.collection.yml @@ -149,5 +149,8 @@ items: # Telemetry overlays - path: .github/instructions/shared/telemetry-overlay.instructions.md kind: instruction + # Hooks + - path: .github/hooks/telemetry.json + kind: hook display: ordering: manual diff --git a/docs/contributing/README.md b/docs/contributing/README.md index f6010a00a..e40c9acf6 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -25,6 +25,7 @@ Use this table to navigate to the appropriate guide based on what you want to co | Report a bug or suggest an enhancement | [CONTRIBUTING.md](https://github.com/microsoft/hve-core/blob/main/CONTRIBUTING.md#reporting-bugs) | | Submit a code change | [CONTRIBUTING.md](https://github.com/microsoft/hve-core/blob/main/CONTRIBUTING.md#your-first-code-contribution) | | Improve general documentation | [CONTRIBUTING.md](https://github.com/microsoft/hve-core/blob/main/CONTRIBUTING.md#improving-the-documentation) | +| Create or modify a hook | [Hooks](hooks.md) | | Create or modify an agent | [Custom Agents](custom-agents.md) | | Create or modify an instruction file | [Instructions](instructions.md) | | Create or modify a prompt | [Prompts](prompts.md) | @@ -44,6 +45,7 @@ Use this table to navigate to the appropriate guide based on what you want to co | [Instructions](instructions.md) | How to create repository-specific coding guidelines | | [Prompts](prompts.md) | How to create reusable prompt templates | | [Skills](skills.md) | How to create skill packages with scripts and documentation | +| [Hooks](hooks.md) | How to implement hook manifests, scripts, and collection registration | | [Release Process](release-process.md) | Extension channels, maturity levels, and publishing workflow | | [Evals in CI](evals-ci.md) | Auth contract, fork-PR policy, and how to add a new eval spec | diff --git a/docs/contributing/hooks.md b/docs/contributing/hooks.md new file mode 100644 index 000000000..e2904a175 --- /dev/null +++ b/docs/contributing/hooks.md @@ -0,0 +1,123 @@ +--- +title: Contributing Hooks +description: How to implement, register, and validate hook artifacts in hve-core +sidebar_position: 7 +author: Microsoft +ms.date: 2026-06-08 +ms.topic: how-to +keywords: + - hooks + - contributing + - telemetry + - sidecar automation +estimated_reading_time: 6 +--- + +## Why Hooks Exist + +Hooks let you run lightweight automation during Copilot lifecycle events without modifying agents, prompts, or skills. In hve-core, hooks are packaged as collection artifacts and can be distributed with other AI customization files. + +Use a hook when you need event-driven behavior such as: + +* collecting local diagnostics or telemetry +* enforcing lightweight local policy checks +* triggering sidecar automation before or after tool calls + +## Hook Layout in This Repository + +Use this structure for hook contributions: + +| Path | Purpose | +|---|---| +| `.github/hooks/.json` | Hook manifest that maps lifecycle events to executable commands | +| `.github/hooks//` | Hook implementation scripts and support files | +| `collections/*.collection.yml` | Collection registration with `kind: hook` | +| `collections/*.collection.md` | Human-readable hook entry in the collection documentation table | + +The telemetry hook is the current reference implementation: + +* `.github/hooks/telemetry.json` +* `.github/hooks/telemetry/` + +## Implementing a New Hook + +1. Add a manifest at `.github/hooks/.json`. +2. Add executable scripts under `.github/hooks//`. +3. Register the hook in one or more `collections/*.collection.yml` files. +4. Document the hook in the matching `collections/*.collection.md` files. +5. Add or update docs under `docs/` for setup and usage. + +Minimal manifest pattern: + +```json +{ + "version": 1, + "hooks": { + "preToolUse": [ + { + "type": "command", + "bash": ".github/hooks/my-hook/my-hook.sh", + "powershell": ".github/hooks/my-hook/Invoke-MyHook.ps1", + "timeoutSec": 10 + } + ] + } +} +``` + +## Script Contract and Runtime Behavior + +For reliability and portability, hook scripts should follow these rules: + +* Read event payload JSON from stdin. +* Return quickly on the disabled path. +* Write `{"continue":true}` to stdout on normal completion. +* Avoid interactive prompts. +* Keep runtime short and respect `timeoutSec` values in the manifest. +* Support both bash and PowerShell paths when practical. + +Telemetry follows this model with a no-op gate and structured JSONL append behavior. + +## Event Compatibility Guidance + +The telemetry manifest includes both lowercase and PascalCase event names to support multiple invocation surfaces. If you need broad compatibility across environments, mirror that pattern in your hook manifest. + +Examples from telemetry: + +* `sessionStart` and `SessionStart` +* `preToolUse` and `PreToolUse` +* `agentStop` and `Stop` + +## Registering a Hook in Collections + +Add a collection item with `kind: hook`: + +```yaml +items: + - path: .github/hooks/my-hook.json + kind: hook +``` + +Then update the corresponding collection markdown (`collections/*.collection.md`) in the Hooks section so users can discover what the hook does. + +## Validation Checklist + +Before opening a PR: + +1. Run `npm run plugin:validate` +2. Run `npm run plugin:generate` +3. Run `npm run lint:md` + +When your hook includes scripts, also run the relevant script linters and tests for those languages. + +## Related Guides + +* [Custom Agents](custom-agents) +* [Instructions](instructions) +* [Common Standards](ai-artifacts-common) +* [Local Telemetry](../customization/local-telemetry) + +--- + +*🤖 Crafted with precision by ✨Copilot following brilliant human instruction, +then carefully refined by our team of discerning human reviewers.* diff --git a/docs/customization/README.md b/docs/customization/README.md index b50f41bfe..8082170ed 100644 --- a/docs/customization/README.md +++ b/docs/customization/README.md @@ -98,7 +98,7 @@ Each HVE role benefits from different customization techniques. The table below | Tech Lead / Architect | [Instructions](instructions.md), [Agents](custom-agents.md), [Skills](skills.md) | Standards enforcement, architecture review agents, and deep domain knowledge | | Security Architect | [Skills](skills.md), [Instructions](instructions.md) | Compliance knowledge packages and security-focused coding conventions | | Data Scientist | [Skills](skills.md), [Prompts](prompts.md) | Analytical domain bundles and repeatable notebook workflows | -| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md) | Infrastructure conventions and DevContainer tuning | +| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md), [Local Telemetry](local-telemetry.md) | Infrastructure conventions, DevContainer tuning, and local telemetry workflows | | Business Program Manager | [Prompts](prompts.md), [Team Adoption](team-adoption.md) | Sprint-planning prompts and governance patterns for stakeholder alignment | | New Contributor | [Instructions](instructions.md), [Environment](environment.md) | Quick onboarding through conventions and a ready-to-use development environment | | Utility | [Collections](collections.md), [Build System](build-system.md) | Cross-cutting tooling assembly and validation pipeline customization | @@ -114,6 +114,7 @@ Each HVE role benefits from different customization techniques. The table below 7. [Forking and Extending](forking.md): Full fork-and-extend customization 8. [Environment Customization](environment.md): DevContainers, VS Code settings, MCP servers 9. [Team Adoption and Governance](team-adoption.md): Governance, naming, onboarding, change management +10. [Local Telemetry](local-telemetry.md): Enable local telemetry, review capture and storage schema mechanics, and generate reports ## Related Resources diff --git a/docs/customization/local-telemetry.md b/docs/customization/local-telemetry.md new file mode 100644 index 000000000..d66fb4f85 --- /dev/null +++ b/docs/customization/local-telemetry.md @@ -0,0 +1,242 @@ +--- +title: Local Telemetry +description: Enable local Copilot session telemetry, understand capture mechanics, and generate local reports +sidebar_position: 10 +author: Microsoft +ms.date: 2026-06-08 +ms.topic: how-to +keywords: + - telemetry + - hooks + - local reporting + - copilot +estimated_reading_time: 7 +--- + +## What This Captures + +The local telemetry hook captures Copilot lifecycle events into local JSONL files. It is intended for local analysis and troubleshooting of your own sessions. + +The telemetry manifest is at `.github/hooks/telemetry.json`. + +Events currently captured include: + +* session start +* user prompt submission +* pre-tool and post-tool use +* subagent start and stop +* agent stop and session end +* pre-compact events + +At stop time, telemetry also appends a session summary with model and token usage when available. + +## Enable Local Telemetry + +Telemetry is opt-in. Enable it with either an environment variable or a repository marker file. + +### Option 1: Environment Variable + +```bash +export HVE_TELEMETRY=1 +``` + +```powershell +$env:HVE_TELEMETRY = "1" +``` + +### Option 2: Repository Marker File + +Create `.hve-telemetry` at the repository root: + +```bash +touch .hve-telemetry +``` + +Either option enables collection. If both are absent, the hook exits in no-op mode. + +## View Reports + +Generate a report with the script in ~/.hve (created at session start): + +```bash +bash ~/.hve/generate-report.sh +``` + +Which will generate a `report.generated.html` for viewing. + +The generated report path is printed when report generation completes. + +The report is self-contained: it embeds every selected JSONL file (session +events plus model and token enrichment) inline. Combined cross-project reports +over a long history (`--all-dirs --date all`) can therefore grow to several +megabytes. Narrow the scope with a specific `--date`, or report a single project +without `--all-dirs`, when a smaller file is preferred. + +## Disable Local Telemetry + +Disable collection by removing both enablement gates: + +1. Unset `HVE_TELEMETRY` +2. Remove `.hve-telemetry` from repository root + +## Where Data Is Written + +Default output directory: + +`/.copilot-tracking/telemetry` + +Override with `HVE_TELEMETRY_DIR` when needed. + +Key files and folders: + +| Path | Purpose | +|---|---| +| `sessions-YYYY-MM-DD.jsonl` | Daily event stream with hook events and session summaries | +| `raw-input.jsonl` | First few raw hook payloads for diagnostics | +| `.stacks/` | Per-session agent stack tracking used for attribution | +| `report.generated.html` | Optional self-contained report output | + +## Data Captured and Storage Schema + +This section describes the mechanics of what local telemetry collects and where each class of data comes from. + +### Collection Pipeline + +1. Copilot lifecycle events invoke the telemetry hook from `.github/hooks/telemetry.json`. +2. Shell entry points (`telemetry-collector.sh` and `Invoke-TelemetryCollector.ps1`) enforce opt-in gates. +3. Event payloads are normalized and appended to daily JSONL files. +4. On stop events, a `SessionSummary` record is appended with model and token aggregates when available. + +### Core Record Types + +The daily JSONL stream contains two primary record types: + +| Record Type | Trigger | Purpose | +|---|---|---| +| Hook event records | Session/tool/subagent lifecycle events | Timeline of what happened during a session | +| `SessionSummary` | Stop event (`Stop`) | Aggregated usage totals and model-level summary | + +### Common Fields + +Most hook event records include: + +* `ts`: Event timestamp (ISO 8601) +* `sid`: Session identifier +* `event`: Canonical event name (for example, `PreToolUse`, `PostToolUse`, `SessionStart`) +* `cwd`: Working directory at capture time + +Additional fields are event-specific. Examples: + +* Prompt events: truncated prompt preview +* Tool events: tool name, selected input keys, response length, inferred agent attribution +* Subagent events: agent name and display name +* Stop events: stop reason + +### Session Summary Fields + +When available at stop time, `SessionSummary` includes: + +* `models`: Model usage map observed during the session +* `model_usage`: Per-model aggregate usage counters +* `input_tokens`, `output_tokens` +* `cache_read_tokens`, `cache_write_tokens` +* `total_nano_aiu` +* `turns`, `messages` +* Optional `reasoning_effort`, `subagent_map`, and `client` + +### Data Sources by Layer + +| Data Category | Source | +|---|---| +| Hook lifecycle events | Copilot hook payloads routed through collector scripts | +| Session summaries | `.copilot/session-state//events.jsonl` (CLI session state) | +| Additional model/token enrichment for reports | VS Code debug logs and session-state aggregation during report generation | + +### Event Naming Normalization + +The pipeline normalizes different casing variants of event names to canonical names used in stored telemetry records. This keeps mixed-client event surfaces queryable from one schema. + +### Storage and Retention Behavior + +* Data is stored locally under `.copilot-tracking/telemetry` by default. +* Records append to date-partitioned files (`sessions-YYYY-MM-DD.jsonl`). +* A small raw payload diagnostic sample is stored in `raw-input.jsonl`. +* Per-session agent stack files are maintained under `.stacks/` for attribution and cleaned up on session stop. + +## Generate a Report + +Use the repository script: + +```bash +npm run telemetry:report +``` + +The script wraps `.github/hooks/telemetry/generate-telemetry-report.sh` and creates a self-contained report HTML file. + +Useful options: + +```bash +bash .github/hooks/telemetry/generate-telemetry-report.sh --help +bash .github/hooks/telemetry/generate-telemetry-report.sh --date all +bash .github/hooks/telemetry/generate-telemetry-report.sh --open +``` + +## Cross-Project Reports + +Telemetry is captured per project, so each repository keeps its own store under +`/.copilot-tracking/telemetry`. To view sessions across every project in a +single report, each store is recorded once per session in a user-level registry +at `~/.hve/telemetry-dirs.txt` (honoring `HVE_HOME`). + +Generate a combined, cross-project report with `--all-dirs`: + +```bash +bash .github/hooks/telemetry/generate-telemetry-report.sh --all-dirs --date all +``` + +The registry self-populates as you work across repositories, so no manual setup +is required. Stale directories (deleted or moved repositories) are pruned +automatically when the report runs. Each session is labeled with its originating +project in the report, so combined output still reads per project. + +## Reports Without the Repository (Extension Users) + +When telemetry runs from the VS Code extension rather than this repository, the +`npm run telemetry:report` script is not present and the report generator lives +at a version-pinned extension path that is awkward to locate. To bridge this, a +cross-project launcher is written into the HVE home directory (`~/.hve`, honoring +`HVE_HOME`) at session start, next to the registry it reads: + +* `~/.hve/generate-report.sh`: for unix shells and Git Bash on Windows +* `~/.hve/generate-report.ps1`: for PowerShell (requires `bash`, for example Git Bash) + +Run the launcher from the HVE home directory without knowing the extension path. +It defaults to a combined, cross-project report written to +`~/.hve/report.generated.html`: + +```bash +bash ~/.hve/generate-report.sh +bash ~/.hve/generate-report.sh --date all +``` + +The launchers are regenerated every session, so they self-heal after an +extension upgrade. They forward any extra arguments to the report generator. + +## Troubleshooting + +Common issues: + +* No events captured: verify one enablement gate is set and your hook manifest is active. +* No enrichment data: model and token enrichment depends on available debug logs and session-state data. +* Report generation fails: install `jq` and ensure `python3` is available. + +## Related Guides + +* [Contributing Hooks](../contributing/hooks) +* [Environment Customization](environment) +* [Managing Collections](collections) + +--- + +*🤖 Crafted with precision by ✨Copilot following brilliant human instruction, +then carefully refined by our team of discerning human reviewers.* diff --git a/plugins/hve-core-all/.github/plugin/plugin.json b/plugins/hve-core-all/.github/plugin/plugin.json index f39e8a684..2355c5deb 100644 --- a/plugins/hve-core-all/.github/plugin/plugin.json +++ b/plugins/hve-core-all/.github/plugin/plugin.json @@ -49,5 +49,6 @@ "skills/rpi/", "skills/security/", "skills/shared/" - ] + ], + "hooks": "hooks/telemetry.json" } \ No newline at end of file diff --git a/plugins/hve-core-all/README.md b/plugins/hve-core-all/README.md index 18a1943ec..6874b936f 100644 --- a/plugins/hve-core-all/README.md +++ b/plugins/hve-core-all/README.md @@ -284,6 +284,12 @@ Use this edition when you want access to everything without choosing a focused c | **video-to-gif** | Video-to-GIF conversion with FFmpeg two-pass optimization | | **vscode-playwright** | VS Code screenshot capture using Playwright MCP with serve-web for slide decks and documentation | +### Hooks + +| Name | Description | +|---------------|----------------------------------------------------------------------------| +| **telemetry** | Records Copilot session lifecycle events to local telemetry for reporting. | + ## Install diff --git a/plugins/hve-core-all/hooks/telemetry b/plugins/hve-core-all/hooks/telemetry new file mode 120000 index 000000000..52d07b1e3 --- /dev/null +++ b/plugins/hve-core-all/hooks/telemetry @@ -0,0 +1 @@ +../../../.github/hooks/telemetry \ No newline at end of file diff --git a/plugins/hve-core-all/hooks/telemetry.json b/plugins/hve-core-all/hooks/telemetry.json new file mode 100644 index 000000000..285749377 --- /dev/null +++ b/plugins/hve-core-all/hooks/telemetry.json @@ -0,0 +1,142 @@ +{ + "version": 1, + "description": "Records Copilot session lifecycle events to local telemetry for reporting.", + "hooks": { + "sessionStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SessionStart": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "userPromptSubmitted": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "UserPromptSubmit": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PreToolUse": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "postToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PostToolUse": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SubagentStart": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SubagentStop": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "agentStop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "Stop": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "sessionEnd": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preCompact": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PreCompact": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ] + } +} diff --git a/plugins/hve-core/.github/plugin/plugin.json b/plugins/hve-core/.github/plugin/plugin.json index d626c91fb..f5efc341d 100644 --- a/plugins/hve-core/.github/plugin/plugin.json +++ b/plugins/hve-core/.github/plugin/plugin.json @@ -13,5 +13,6 @@ "skills/experimental/", "skills/hve-core/", "skills/shared/" - ] + ], + "hooks": "hooks/telemetry.json" } \ No newline at end of file diff --git a/plugins/hve-core/README.md b/plugins/hve-core/README.md index 619a0720a..3c02d9755 100644 --- a/plugins/hve-core/README.md +++ b/plugins/hve-core/README.md @@ -88,6 +88,12 @@ HVE Core provides the flagship RPI (Research, Plan, Implement, Review) workflow | **telemetry-foundations** | Declarative OpenTelemetry-aligned telemetry vocabulary and instrumentation conventions for traces, metrics, logs, and PII handling | | **vally-tests** | Authors Vally conformance tests for prompts, instructions, agents, and skills, including refusals for jailbreak, prompt-injection, harmful-elicitation, TOS, CoC, and PII-extraction stimuli | +### Hooks + +| Name | Description | +|---------------|----------------------------------------------------------------------------| +| **telemetry** | Records Copilot session lifecycle events to local telemetry for reporting. | + ## Install diff --git a/plugins/hve-core/hooks/telemetry b/plugins/hve-core/hooks/telemetry new file mode 120000 index 000000000..52d07b1e3 --- /dev/null +++ b/plugins/hve-core/hooks/telemetry @@ -0,0 +1 @@ +../../../.github/hooks/telemetry \ No newline at end of file diff --git a/plugins/hve-core/hooks/telemetry.json b/plugins/hve-core/hooks/telemetry.json new file mode 100644 index 000000000..285749377 --- /dev/null +++ b/plugins/hve-core/hooks/telemetry.json @@ -0,0 +1,142 @@ +{ + "version": 1, + "description": "Records Copilot session lifecycle events to local telemetry for reporting.", + "hooks": { + "sessionStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SessionStart": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "userPromptSubmitted": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "UserPromptSubmit": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PreToolUse": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "postToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PostToolUse": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SubagentStart": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "SubagentStop": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "agentStop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "Stop": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "sessionEnd": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preCompact": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "PreCompact": [ + { + "type": "command", + "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", + "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ] + } +} diff --git a/scripts/collections/Modules/CollectionHelpers.psm1 b/scripts/collections/Modules/CollectionHelpers.psm1 index dd9b9ff4d..340c2681f 100644 --- a/scripts/collections/Modules/CollectionHelpers.psm1 +++ b/scripts/collections/Modules/CollectionHelpers.psm1 @@ -223,6 +223,9 @@ function Get-CollectionArtifactKey { 'skill' { return [System.IO.Path]::GetFileName($Path.TrimEnd('/')) } + 'hook' { + return [System.IO.Path]::GetFileNameWithoutExtension($Path.TrimEnd('/')) + } default { if ($Path -match "\.$([regex]::Escape($Kind))\.md$") { return ([System.IO.Path]::GetFileName($Path) -replace "\.$([regex]::Escape($Kind))\.md$", '') @@ -361,7 +364,7 @@ function Get-ArtifactFiles { .DESCRIPTION Scans .github/agents/, .github/prompts/, .github/instructions/ (recursively), - and .github/skills/ to build a complete list of collection items. Returns + .github/skills/, and .github/hooks/ to build a complete list of collection items. Returns repo-relative paths with forward slashes. .PARAMETER RepoRoot @@ -427,6 +430,24 @@ function Get-ArtifactFiles { } } + # Hooks (JSON files under .github/hooks/) + $hooksDir = Join-Path -Path $RepoRoot -ChildPath '.github/hooks' + if (Test-Path -Path $hooksDir) { + # Hook manifests live at the top level (.github/hooks/.json); + # implementation files under .github/hooks// must not be treated + # as manifests, so do not recurse. + $hookFiles = Get-ChildItem -Path $hooksDir -Filter '*.json' -File + foreach ($hookFile in $hookFiles) { + $relativePath = [System.IO.Path]::GetRelativePath($RepoRoot, $hookFile.FullName) -replace '\\', '/' + + if (Test-DeprecatedPath -Path $relativePath) { + continue + } + + $items += @{ path = $relativePath; kind = 'hook' } + } + } + return $items } @@ -700,6 +721,27 @@ function Get-ArtifactDescription { return '' } + # Hook manifests are JSON with no frontmatter; read their top-level + # description field instead of scanning for a YAML block. + if ([System.IO.Path]::GetExtension($FilePath) -eq '.json') { + try { + # Hook manifests can contain keys differing only by case (e.g. + # sessionStart / SessionStart), which ConvertFrom-Json rejects when + # building a case-insensitive PSCustomObject. -AsHashtable uses a + # case-sensitive map that tolerates those keys; only the top-level + # description is read here. + $json = Get-Content -Path $FilePath -Raw | ConvertFrom-Json -AsHashtable + $desc = $json['description'] + if ($desc) { + return ([string]$desc).Trim() + } + } + catch { + Write-Verbose "Failed to parse JSON description from $FilePath`: $_" + } + return '' + } + $content = Get-Content -Path $FilePath -Raw if ($content -match '(?s)^---\s*\r?\n(.*?)\r?\n---') { $yamlBlock = $Matches[1] diff --git a/scripts/linting/schemas/collection-manifest.schema.json b/scripts/linting/schemas/collection-manifest.schema.json index 53dada86a..c0a68c643 100644 --- a/scripts/linting/schemas/collection-manifest.schema.json +++ b/scripts/linting/schemas/collection-manifest.schema.json @@ -107,4 +107,4 @@ } }, "additionalProperties": false -} \ No newline at end of file +} diff --git a/scripts/plugins/Generate-Plugins.ps1 b/scripts/plugins/Generate-Plugins.ps1 index a5457fa38..8a4774db9 100644 --- a/scripts/plugins/Generate-Plugins.ps1 +++ b/scripts/plugins/Generate-Plugins.ps1 @@ -244,6 +244,7 @@ function Invoke-PluginGeneration { $totalCommands = 0 $totalInstructions = 0 $totalSkills = 0 + $totalHooks = 0 foreach ($collection in $allCollections) { $id = $collection.id @@ -280,6 +281,7 @@ function Invoke-PluginGeneration { $prompts = @() $instructions = @() $skills = @() + $hooks = @() foreach ($item in $filteredCollection.items) { if (-not $item.ContainsKey('kind') -or -not $item.ContainsKey('path')) { @@ -301,6 +303,7 @@ function Invoke-PluginGeneration { 'prompt' { $prompts += $entry } 'instruction' { $instructions += $entry } 'skill' { $skills += $entry } + 'hook' { $hooks += $entry } } } @@ -310,7 +313,8 @@ function Invoke-PluginGeneration { @{ Title = 'Chat Agents'; Items = $agents }, @{ Title = 'Prompts'; Items = $prompts }, @{ Title = 'Instructions'; Items = $instructions }, - @{ Title = 'Skills'; Items = $skills } + @{ Title = 'Skills'; Items = $skills }, + @{ Title = 'Hooks'; Items = $hooks } )) { if ($section.Items.Count -eq 0) { continue } @@ -393,6 +397,7 @@ function Invoke-PluginGeneration { $totalCommands += $result.CommandCount $totalInstructions += $result.InstructionCount $totalSkills += $result.SkillCount + $totalHooks += $result.HookCount $generated++ Write-Host " $id ($itemCount items)" -ForegroundColor Green @@ -419,6 +424,7 @@ function Invoke-PluginGeneration { Write-Host " Commands: $totalCommands" Write-Host " Instructions: $totalInstructions" Write-Host " Skills: $totalSkills" + Write-Host " Hooks: $totalHooks" return New-GenerateResult -Success $true -PluginCount $generated } diff --git a/scripts/plugins/Modules/PluginHelpers.psm1 b/scripts/plugins/Modules/PluginHelpers.psm1 index 547732a17..56be660c6 100644 --- a/scripts/plugins/Modules/PluginHelpers.psm1 +++ b/scripts/plugins/Modules/PluginHelpers.psm1 @@ -41,7 +41,7 @@ function Get-PluginItemName { [string]$FileName, [Parameter(Mandatory = $true)] - [ValidateSet('agent', 'prompt', 'instruction', 'skill')] + [ValidateSet('agent', 'prompt', 'instruction', 'skill', 'hook')] [string]$Kind ) @@ -50,6 +50,7 @@ function Get-PluginItemName { 'prompt' { return $FileName -replace '\.prompt\.md$', '.md' } 'instruction' { return $FileName } 'skill' { return $FileName } + 'hook' { return $FileName } } } @@ -79,7 +80,7 @@ function Get-PluginItemSubpath { [string]$Path, [Parameter(Mandatory = $true)] - [ValidateSet('agent', 'prompt', 'instruction', 'skill')] + [ValidateSet('agent', 'prompt', 'instruction', 'skill', 'hook')] [string]$Kind ) @@ -88,6 +89,7 @@ function Get-PluginItemSubpath { 'prompt' = '.github/prompts/' 'instruction' = '.github/instructions/' 'skill' = '.github/skills/' + 'hook' = '.github/hooks/' } $prefix = $prefixMap[$Kind] @@ -126,7 +128,7 @@ function Get-PluginSubdirectory { [OutputType([string])] param( [Parameter(Mandatory = $true)] - [ValidateSet('agent', 'prompt', 'instruction', 'skill')] + [ValidateSet('agent', 'prompt', 'instruction', 'skill', 'hook')] [string]$Kind ) @@ -135,6 +137,7 @@ function Get-PluginSubdirectory { 'prompt' { return 'commands' } 'instruction' { return 'instructions' } 'skill' { return 'skills' } + 'hook' { return 'hooks' } } } @@ -168,6 +171,9 @@ function New-PluginManifestContent { .PARAMETER SkillPaths Optional. Array of relative directory paths containing skill subdirs. + .PARAMETER HookPaths + Optional. Array of relative file paths to hook JSON files. + .OUTPUTS [hashtable] Plugin manifest with name, description, version, and component path keys. @@ -194,7 +200,11 @@ function New-PluginManifestContent { [Parameter(Mandatory = $false)] [AllowEmptyCollection()] - [string[]]$SkillPaths + [string[]]$SkillPaths, + + [Parameter(Mandatory = $false)] + [AllowEmptyCollection()] + [string[]]$HookPaths ) $manifest = [ordered]@{ @@ -217,6 +227,17 @@ function New-PluginManifestContent { $manifest['skills'] = @($SkillPaths | Sort-Object) } + if ($HookPaths -and $HookPaths.Count -gt 0) { + # The CLI `hooks` field is a single hooks-config file path (or inline + # object), not an array. Emit the lone path as a string; warn when more + # than one hook manifest is registered since only one can be referenced. + $sortedHooks = @($HookPaths | Sort-Object) + if ($sortedHooks.Count -gt 1) { + Write-Warning "Plugin '$CollectionId' declares $($sortedHooks.Count) hook manifests; the CLI references only one. Using '$($sortedHooks[0])'." + } + $manifest['hooks'] = $sortedHooks[0] + } + return $manifest } @@ -320,6 +341,7 @@ function New-PluginReadmeContent { prompt = @{ Title = 'Commands'; Header = 'Command' } instruction = @{ Title = 'Instructions'; Header = 'Instruction' } skill = @{ Title = 'Skills'; Header = 'Skill' } + hook = @{ Title = 'Hooks'; Header = 'Hook' } } $hasCollectionArtifactContent = -not [string]::IsNullOrWhiteSpace($CollectionContent) -and ( @@ -635,6 +657,71 @@ function New-PluginLink { } } +function Write-PluginHookArtifact { + <# + .SYNOPSIS + Materializes a hook manifest and its sibling script directory into a plugin. + + .DESCRIPTION + Hook command paths in the source manifest are repository-root relative + (for example .github/hooks/telemetry/telemetry-collector.sh) so they resolve + when the hook is auto-loaded from a checked-out repository. Inside an + installed plugin the same scripts live under the plugin root, so this + function writes a transformed copy of the manifest with those paths + rewritten to the ${PLUGIN_ROOT} placeholder, then links the sibling script + directory (the manifest path without its .json extension) alongside it. + + .PARAMETER SourceManifest + Absolute path to the source hook .json manifest in the repository. + + .PARAMETER DestinationManifest + Absolute path where the transformed manifest is written in the plugin. + + .PARAMETER GeneratedFiles + Set tracking generated paths for orphan cleanup; the linked script + directory is added to it. + + .PARAMETER SymlinkCapable + When set, links the script directory; otherwise writes a text stub. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$SourceManifest, + + [Parameter(Mandatory = $true)] + [string]$DestinationManifest, + + [Parameter(Mandatory = $true)] + [System.Collections.Generic.HashSet[string]]$GeneratedFiles, + + [Parameter(Mandatory = $false)] + [switch]$SymlinkCapable + ) + + # Degrade gracefully when the manifest is missing, matching how other kinds + # warn rather than throw and fail the entire generation run. + if (-not (Test-Path -LiteralPath $SourceManifest)) { + Write-Warning "Hook manifest not found: $SourceManifest" + return + } + + # Rewrite repo-root-relative hook script paths to plugin-relative paths so + # commands resolve from the installed plugin directory. Literal string + # replacement avoids regex interpretation of the path and the $ placeholder. + $manifestText = Get-Content -LiteralPath $SourceManifest -Raw -Encoding utf8 + $manifestText = $manifestText.Replace('.github/hooks/', '${PLUGIN_ROOT}/hooks/') + Set-ContentIfChanged -Path $DestinationManifest -Value $manifestText | Out-Null + + # Link the sibling script directory (manifest path without .json extension). + $scriptSrc = $SourceManifest -replace '\.json$', '' + if (Test-Path -LiteralPath $scriptSrc) { + $scriptDest = $DestinationManifest -replace '\.json$', '' + [void]$GeneratedFiles.Add($scriptDest) + New-PluginLink -SourcePath $scriptSrc -DestinationPath $scriptDest -SymlinkCapable:$SymlinkCapable + } +} + function Write-PluginDirectory { <# .SYNOPSIS @@ -707,6 +794,7 @@ function Write-PluginDirectory { CommandCount = 0 InstructionCount = 0 SkillCount = 0 + HookCount = 0 } # Track unique directories per kind for plugin.json path arrays @@ -719,6 +807,9 @@ function Write-PluginDirectory { $skillDirs = [System.Collections.Generic.HashSet[string]]::new( [System.StringComparer]::OrdinalIgnoreCase ) + $hookFiles = [System.Collections.Generic.HashSet[string]]::new( + [System.StringComparer]::OrdinalIgnoreCase + ) $readmeItems = @() $generatedFiles = [System.Collections.Generic.HashSet[string]]::new( @@ -761,20 +852,25 @@ function Write-PluginDirectory { $destPath = Join-Path -Path $pluginRoot -ChildPath $subdir -AdditionalChildPath $itemName } - # Read frontmatter from the source file for description - $fallback = $itemName -replace '\.md$', '' - if (Test-Path -Path $sourcePath) { - $frontmatter = Get-ArtifactFrontmatter -FilePath $sourcePath -FallbackDescription $fallback - $description = $frontmatter.description - } - else { + # Read description from the source file. Hook manifests are JSON + # with no frontmatter, so read their top-level description field. + $fallback = $itemName -replace '\.(md|json)$', '' + if (-not (Test-Path -Path $sourcePath)) { $description = $fallback Write-Warning "Source file not found: $sourcePath" } + elseif ($kind -eq 'hook') { + $hookDesc = Get-ArtifactDescription -FilePath $sourcePath + $description = if ($hookDesc) { $hookDesc } else { $fallback } + } + else { + $frontmatter = Get-ArtifactFrontmatter -FilePath $sourcePath -FallbackDescription $fallback + $description = $frontmatter.description + } } $readmeItems += @{ - Name = $itemName -replace '\.md$', '' + Name = ($itemName -replace '\.md$', '') -replace '\.json$', '' Description = $description Kind = $kind } @@ -801,6 +897,11 @@ function Write-PluginDirectory { $relDir = [System.IO.Path]::GetRelativePath($pluginRoot, $parentDir) -replace '\\', '/' [void]$skillDirs.Add("$relDir/") } + 'hook' { + $counts.HookCount++ + $relPath = [System.IO.Path]::GetRelativePath($pluginRoot, $destPath) -replace '\\', '/' + [void]$hookFiles.Add($relPath) + } } [void]$generatedFiles.Add($destPath) @@ -810,7 +911,15 @@ function Write-PluginDirectory { continue } - New-PluginLink -SourcePath $sourcePath -DestinationPath $destPath -SymlinkCapable:$SymlinkCapable + # Hooks bundle a sibling script directory and need plugin-relative + # command paths; other kinds link their single source file directly. + if ($kind -eq 'hook') { + Write-PluginHookArtifact -SourceManifest $sourcePath -DestinationManifest $destPath ` + -GeneratedFiles $generatedFiles -SymlinkCapable:$SymlinkCapable + } + else { + New-PluginLink -SourcePath $sourcePath -DestinationPath $destPath -SymlinkCapable:$SymlinkCapable + } } # Link shared resource directories (unconditional, all plugins) @@ -847,7 +956,8 @@ function Write-PluginDirectory { -Version $Version ` -AgentPaths @($agentDirs) ` -CommandPaths @($commandDirs) ` - -SkillPaths @($skillDirs) + -SkillPaths @($skillDirs) ` + -HookPaths @($hookFiles) [void]$generatedFiles.Add($manifestPath) if ($DryRun) { @@ -883,6 +993,7 @@ function Write-PluginDirectory { CommandCount = $counts.CommandCount InstructionCount = $counts.InstructionCount SkillCount = $counts.SkillCount + HookCount = $counts.HookCount GeneratedFiles = $generatedFiles } } From 5d38332e3bff8ab77881b72893ce1776426c1dcc Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Mon, 15 Jun 2026 16:30:10 -0600 Subject: [PATCH 02/10] powershell path for report generation --- .../telemetry/Invoke-TelemetryReport.ps1 | 223 ++++++++++++++++++ .github/hooks/telemetry/_telemetry_core.py | 32 +-- .github/hooks/telemetry/clean-telemetry.sh | 1 - .../telemetry/generate-telemetry-report.sh | 1 - .github/hooks/telemetry/report.html | 6 +- .../telemetry/tests/test_telemetry_core.py | 8 +- docs/customization/local-telemetry.md | 42 ++-- 7 files changed, 275 insertions(+), 38 deletions(-) create mode 100644 .github/hooks/telemetry/Invoke-TelemetryReport.ps1 diff --git a/.github/hooks/telemetry/Invoke-TelemetryReport.ps1 b/.github/hooks/telemetry/Invoke-TelemetryReport.ps1 new file mode 100644 index 000000000..c31e8a52c --- /dev/null +++ b/.github/hooks/telemetry/Invoke-TelemetryReport.ps1 @@ -0,0 +1,223 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +#Requires -Version 7.0 + +<# +.SYNOPSIS + Generates a self-contained telemetry report from Copilot hook JSONL files. +.DESCRIPTION + Native PowerShell counterpart to generate-telemetry-report.sh. Discovers the + session files for a target date, embeds their contents into a copy of + report.html, and writes a self-contained report.generated.html that renders + automatically without drag & drop. + + Orchestration (file discovery, JSON embedding, template injection) is native + PowerShell so Windows hosts need no bash. Token/model enrichment is delegated + to the shared Python engine (_telemetry_core.py) via its aggregate-debug, + aggregate-session, and list-dirs modes - the same modes the bash entry point + uses - so the enrichment logic stays single-sourced across platforms. Python + is optional: when absent, the report is produced without enrichment. +.PARAMETER Date + Target date (yyyy-MM-dd). Default: today (UTC). Use 'all' to include every + sessions-*.jsonl file. +.PARAMETER AllDirs + Scan every per-project telemetry directory recorded in the user-level + registry (~/.copilot/telemetry-dirs.txt) for a combined cross-project report. +.PARAMETER Path + Telemetry directory. Default: /.copilot-tracking/telemetry +.PARAMETER DebugLog + Optional debug log JSONL (e.g. main.jsonl) for token data. When omitted, VS + Code debug logs are auto-discovered and the precise model version plus token + data are joined in by session id. +.PARAMETER Output + Output path. Default: /report.generated.html +.PARAMETER Open + Open the generated report in the default browser. +.NOTES + Runs via: pwsh Invoke-TelemetryReport.ps1 +#> +[CmdletBinding()] +param( + [Alias('d')] + [string]$Date, + [Alias('a')] + [switch]$AllDirs, + [Alias('p')] + [string]$Path, + [Alias('l')] + [string]$DebugLog, + [Alias('o')] + [string]$Output, + [switch]$Open +) + +$ErrorActionPreference = 'Stop' + +$TemplatePath = Join-Path $PSScriptRoot 'report.html' +$CorePy = Join-Path $PSScriptRoot '_telemetry_core.py' + +#region Resolve repo root +$RepoRoot = $env:HVE_REPO_ROOT +if (-not $RepoRoot -and (Get-Command git -ErrorAction SilentlyContinue)) { + try { $RepoRoot = & git -C $PSScriptRoot rev-parse --show-toplevel 2>$null } catch { $RepoRoot = $null } +} +if (-not $RepoRoot) { + $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +} +#endregion Resolve repo root + +# Python enables best-effort enrichment only; the report works without it. +$Python = Get-Command python3 -ErrorAction SilentlyContinue +if (-not $Python) { + $Python = Get-Command python -ErrorAction SilentlyContinue +} + +# Emit the user-level registry of per-project telemetry directories (one path +# per line). Used by -AllDirs for cross-project reports. Empty without Python. +function Get-RegistryDir { + if (-not $Python) { return @() } + try { + $lines = & $Python.Source $CorePy list-dirs 2>$null + if ($LASTEXITCODE -eq 0 -and $lines) { + return @($lines | Where-Object { $_ -and $_.Trim() }) + } + } catch { + Write-Verbose "Registry lookup failed; continuing without cross-project dirs: $_" + } + return @() +} + +# Best-effort enrichment: extract llm_request events (precise model, token, and +# duration data) from VS Code debug logs, scoped to the supplied hook files. +# Returns $true when matching events were written to $OutFile. +function Invoke-AggregateDebug { + param([string]$OutFile, [string[]]$HookFile) + if (-not $Python) { return $false } + & $Python.Source $CorePy aggregate-debug $OutFile @HookFile 2>$null + return ($LASTEXITCODE -eq 0) +} + +# Enrichment from CLI session state: produces llm_request-compatible events so +# CLI sessions without VS Code debug logs still show model/token/duration data. +function Invoke-AggregateSession { + param([string]$OutFile, [string[]]$HookFile) + if (-not $Python) { return $false } + & $Python.Source $CorePy aggregate-session $OutFile @HookFile 2>$null + return ($LASTEXITCODE -eq 0) +} + +if (-not (Test-Path -LiteralPath $TemplatePath)) { + Write-Error "Template not found: $TemplatePath" + exit 1 +} + +$TargetDate = if ($Date) { $Date } else { (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd') } +$TelemetryPath = if ($Path) { $Path } else { Join-Path $RepoRoot '.copilot-tracking/telemetry' } +$OutputPath = if ($Output) { $Output } else { Join-Path $TelemetryPath 'report.generated.html' } + +# Determine which telemetry directories to scan. With -AllDirs, prepend every +# directory recorded in the user-level registry (cross-project view). +$SearchDirs = [System.Collections.Generic.List[string]]::new() +if ($AllDirs) { + foreach ($d in (Get-RegistryDir)) { $SearchDirs.Add($d) } +} +$SearchDirs.Add($TelemetryPath) + +# Collect session files for the target date across the chosen directories, +# de-duplicating directories that appear more than once. +$Pattern = if ($TargetDate -eq 'all') { 'sessions-*.jsonl' } else { "sessions-$TargetDate.jsonl" } +$Files = [System.Collections.Generic.List[string]]::new() +$SeenDirs = [System.Collections.Generic.HashSet[string]]::new() +foreach ($dir in $SearchDirs) { + if (-not $dir -or -not $SeenDirs.Add($dir)) { continue } + if (-not (Test-Path -LiteralPath $dir -PathType Container)) { continue } + Get-ChildItem -LiteralPath $dir -Filter $Pattern -File -ErrorAction SilentlyContinue | + Sort-Object Name | + ForEach-Object { $Files.Add($_.FullName) } +} + +# Temp files for enrichment payloads; cleaned up on exit. +$TmpDir = $null +try { + if ($DebugLog) { + if (-not (Test-Path -LiteralPath $DebugLog)) { + Write-Error "Debug log not found: $DebugLog" + exit 1 + } + $Files.Add((Resolve-Path -LiteralPath $DebugLog).Path) + } elseif ($Files.Count -gt 0) { + # Auto-enrich with precise model + token data from VS Code debug logs and + # CLI session state, scoped to the sessions already collected. + $TmpDir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path $TmpDir -Force | Out-Null + + $DebugAgg = Join-Path $TmpDir 'debug-llm-requests.jsonl' + if (Invoke-AggregateDebug -OutFile $DebugAgg -HookFile $Files.ToArray()) { + $Files.Add($DebugAgg) + } + + $CliAgg = Join-Path $TmpDir 'cli-session-state.jsonl' + if (Invoke-AggregateSession -OutFile $CliAgg -HookFile $Files.ToArray()) { + $Files.Add($CliAgg) + } + } + + if ($Files.Count -eq 0) { + Write-Warning "No telemetry files found in '$TelemetryPath' for date '$TargetDate'." + exit 0 + } + + # Build a JSON array of {name, content} objects. ConvertTo-Json escapes '<' + # to \u003c, so no literal survives in the embedded text; the + # explicit ' + $TagClose = '' + $Lines = (Get-Content -LiteralPath $TemplatePath -Raw) -split "`n" + $Builder = [System.Text.StringBuilder]::new() + $Injected = $false + for ($i = 0; $i -lt $Lines.Count; $i++) { + if (-not $Injected -and $Lines[$i].Contains('id="embeddedData"')) { + [void]$Builder.Append($TagOpen).Append($Data).Append($TagClose) + $Injected = $true + } else { + [void]$Builder.Append($Lines[$i]) + } + if ($i -lt $Lines.Count - 1) { [void]$Builder.Append("`n") } + } + + $OutDir = Split-Path -Parent $OutputPath + if ($OutDir -and -not (Test-Path -LiteralPath $OutDir)) { + New-Item -ItemType Directory -Path $OutDir -Force | Out-Null + } + [System.IO.File]::WriteAllText($OutputPath, $Builder.ToString()) + + Write-Host "Wrote self-contained report: $OutputPath" + $Names = ($Files | ForEach-Object { Split-Path -Leaf $_ }) -join ', ' + Write-Host ("Embedded {0} file(s): {1}" -f $Files.Count, $Names) + + if ($Open) { + if ($IsWindows) { + Start-Process $OutputPath + } elseif ($IsMacOS -and (Get-Command open -ErrorAction SilentlyContinue)) { + & open $OutputPath + } elseif (Get-Command xdg-open -ErrorAction SilentlyContinue) { + & xdg-open $OutputPath + } + } +} finally { + if ($TmpDir -and (Test-Path -LiteralPath $TmpDir)) { + Remove-Item -LiteralPath $TmpDir -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/.github/hooks/telemetry/_telemetry_core.py b/.github/hooks/telemetry/_telemetry_core.py index 06494ea05..60c264c5b 100644 --- a/.github/hooks/telemetry/_telemetry_core.py +++ b/.github/hooks/telemetry/_telemetry_core.py @@ -4,13 +4,12 @@ """Canonical telemetry engine shared by the Copilot hook collectors. This module is the single source of truth for telemetry collection. The bash -collector ``telemetry-collector.sh`` invokes the ``collect`` mode to record one -hook event (and enrich the session at ``Stop``), while -``generate-telemetry-report.sh`` invokes the ``aggregate-debug``, -``aggregate-session``, and ``list-dirs`` modes to join model/token data and -discover per-project telemetry stores for reports. ``clean-telemetry.sh`` -invokes the ``clean`` mode to remove telemetry artifacts from one or every -registered store. +collector invokes the ``collect`` mode to record one hook event (and enrich +the session at ``Stop``), while the report generators invoke the +``aggregate-debug``, ``aggregate-session``, and ``list-dirs`` modes to join +model/token data and discover per-project telemetry stores for reports. +Clean scripts invoke the ``clean`` mode to remove telemetry artifacts +from one or every registered store. The PowerShell collector ``Invoke-TelemetryCollector.ps1`` is a thin wrapper that delegates to this same engine via ``collect``, so the collection logic @@ -166,17 +165,17 @@ def register_telemetry_dir(tel_dir: Path, registry: Path | None = None) -> None: _PWSH_LAUNCHER = """# Generated by HVE telemetry. Regenerated each session; edits will be lost. # Cross-project telemetry report launcher. Lives in the HVE home directory -# alongside the registry of per-project telemetry stores. Requires bash (for -# example Git Bash). Run from this directory: -# ./generate-report.ps1 # today, every project -# ./generate-report.ps1 --date all # every captured day, every project +# alongside the registry of per-project telemetry stores. Runs natively through +# PowerShell. Run from this directory: +# ./generate-report.ps1 # today, every project +# ./generate-report.ps1 -Date all # every captured day, every project $ReportScript = '__REPORT_SCRIPT__' if (-not (Test-Path $ReportScript)) { Write-Error "Telemetry report script not found: $ReportScript" Write-Error 'Start a new Copilot session to regenerate this launcher.' exit 1 } -bash $ReportScript --all-dirs --output '__OUT__' @args +& $ReportScript -AllDirs -Output '__OUT__' @args """ @@ -232,13 +231,14 @@ def write_report_launchers(script_dir: Path | None = None) -> None: cross-project mode so running them spans every project. Only the launchers for the host platform are written: PowerShell (``.ps1``) - on Windows, POSIX shell (``.sh``) elsewhere. The cleanup launcher is fully - native per platform (bash wrapper on POSIX, PowerShell wrapper on Windows), - so no cross-interpreter dependency is required for cleanup. + on Windows, POSIX shell (``.sh``) elsewhere. Both the report and cleanup + launchers are fully native per platform (bash wrappers on POSIX, PowerShell + wrappers on Windows), so no cross-interpreter dependency is required. """ if script_dir is None: script_dir = Path(__file__).resolve().parent report_script = str(script_dir / "generate-telemetry-report.sh") + report_ps1 = str(script_dir / "Invoke-TelemetryReport.ps1") clean_script = str(script_dir / "clean-telemetry.sh") clean_ps1 = str(script_dir / "Invoke-TelemetryClean.ps1") home = hve_home() @@ -248,7 +248,7 @@ def write_report_launchers(script_dir: Path | None = None) -> None: if _is_windows(): # Quote for PowerShell single-quoted strings (double embedded '). pwsh_text = _PWSH_LAUNCHER.replace( - "__REPORT_SCRIPT__", report_script.replace("'", "''") + "__REPORT_SCRIPT__", report_ps1.replace("'", "''") ).replace("__OUT__", out_path.replace("'", "''")) pwsh_clean_text = _PWSH_CLEAN_LAUNCHER.replace( "__CLEAN_PS1__", clean_ps1.replace("'", "''") diff --git a/.github/hooks/telemetry/clean-telemetry.sh b/.github/hooks/telemetry/clean-telemetry.sh index 5386ec711..b5a938aeb 100755 --- a/.github/hooks/telemetry/clean-telemetry.sh +++ b/.github/hooks/telemetry/clean-telemetry.sh @@ -35,7 +35,6 @@ Options: use). -h, --help Show this help. -Runs via: npm run telemetry:clean EOF } diff --git a/.github/hooks/telemetry/generate-telemetry-report.sh b/.github/hooks/telemetry/generate-telemetry-report.sh index 7e4f2edf0..b9979f8d7 100755 --- a/.github/hooks/telemetry/generate-telemetry-report.sh +++ b/.github/hooks/telemetry/generate-telemetry-report.sh @@ -37,7 +37,6 @@ Options: --open Open the generated report in the default browser. -h, --help Show this help. -Runs via: npm run telemetry:report EOF } diff --git a/.github/hooks/telemetry/report.html b/.github/hooks/telemetry/report.html index e95c4c86e..3fdd70fd0 100644 --- a/.github/hooks/telemetry/report.html +++ b/.github/hooks/telemetry/report.html @@ -110,7 +110,8 @@

Local Generated Report

- @@ -170,7 +171,8 @@

Tool Latency

} } - /** Load data embedded by Invoke-TelemetryReport.ps1, if any. */ + /** Load data embedded by the report generator (generate-telemetry-report.sh + or Invoke-TelemetryReport.ps1), if any. */ function loadEmbedded() { const el = document.getElementById('embeddedData'); if (!el) return false; diff --git a/.github/hooks/telemetry/tests/test_telemetry_core.py b/.github/hooks/telemetry/tests/test_telemetry_core.py index 1677971e5..9de886bca 100644 --- a/.github/hooks/telemetry/tests/test_telemetry_core.py +++ b/.github/hooks/telemetry/tests/test_telemetry_core.py @@ -548,12 +548,14 @@ def test_given_windows_when_write_report_launchers_then_writes_ps1_only(tmp_path monkeypatch.setenv("HVE_HOME", str(hve)) monkeypatch.setattr(core, "_is_windows", lambda: True) core.write_report_launchers(script_dir) - report_script = str(script_dir / "generate-telemetry-report.sh") + report_ps1 = str(script_dir / "Invoke-TelemetryReport.ps1") out_path = str(hve / "report.generated.html") ps = (hve / "generate-report.ps1").read_text(encoding="utf-8") - assert report_script in ps + # Native delegation to the PowerShell generator, no bash. + assert report_ps1 in ps assert out_path in ps - assert "--all-dirs" in ps + assert "-AllDirs" in ps + assert "bash" not in ps # No POSIX launcher on Windows. assert not (hve / "generate-report.sh").exists() diff --git a/docs/customization/local-telemetry.md b/docs/customization/local-telemetry.md index d66fb4f85..7e6c34fd8 100644 --- a/docs/customization/local-telemetry.md +++ b/docs/customization/local-telemetry.md @@ -165,15 +165,7 @@ The pipeline normalizes different casing variants of event names to canonical na ## Generate a Report -Use the repository script: - -```bash -npm run telemetry:report -``` - -The script wraps `.github/hooks/telemetry/generate-telemetry-report.sh` and creates a self-contained report HTML file. - -Useful options: +Run the report generator directly: ```bash bash .github/hooks/telemetry/generate-telemetry-report.sh --help @@ -181,6 +173,13 @@ bash .github/hooks/telemetry/generate-telemetry-report.sh --date all bash .github/hooks/telemetry/generate-telemetry-report.sh --open ``` +On Windows (or any PowerShell host) the native equivalent needs no `bash`: + +```powershell +pwsh .github/hooks/telemetry/Invoke-TelemetryReport.ps1 -Date all +pwsh .github/hooks/telemetry/Invoke-TelemetryReport.ps1 -Open +``` + ## Cross-Project Reports Telemetry is captured per project, so each repository keeps its own store under @@ -194,6 +193,12 @@ Generate a combined, cross-project report with `--all-dirs`: bash .github/hooks/telemetry/generate-telemetry-report.sh --all-dirs --date all ``` +The PowerShell generator takes `-AllDirs` for the same cross-project report: + +```powershell +pwsh .github/hooks/telemetry/Invoke-TelemetryReport.ps1 -AllDirs -Date all +``` + The registry self-populates as you work across repositories, so no manual setup is required. Stale directories (deleted or moved repositories) are pruned automatically when the report runs. Each session is labeled with its originating @@ -202,13 +207,13 @@ project in the report, so combined output still reads per project. ## Reports Without the Repository (Extension Users) When telemetry runs from the VS Code extension rather than this repository, the -`npm run telemetry:report` script is not present and the report generator lives -at a version-pinned extension path that is awkward to locate. To bridge this, a -cross-project launcher is written into the HVE home directory (`~/.hve`, honoring -`HVE_HOME`) at session start, next to the registry it reads: +report generator lives at a version-pinned extension path that is awkward to +locate. To bridge this, a cross-project launcher is written into the HVE home +directory (`~/.hve`, honoring `HVE_HOME`) at session start, next to the registry +it reads: * `~/.hve/generate-report.sh`: for unix shells and Git Bash on Windows -* `~/.hve/generate-report.ps1`: for PowerShell (requires `bash`, for example Git Bash) +* `~/.hve/generate-report.ps1`: for PowerShell, runs natively (no `bash` required) Run the launcher from the HVE home directory without knowing the extension path. It defaults to a combined, cross-project report written to @@ -219,6 +224,13 @@ bash ~/.hve/generate-report.sh bash ~/.hve/generate-report.sh --date all ``` +From PowerShell, run the native launcher: + +```powershell +~/.hve/generate-report.ps1 +~/.hve/generate-report.ps1 -Date all +``` + The launchers are regenerated every session, so they self-heal after an extension upgrade. They forward any extra arguments to the report generator. @@ -228,7 +240,7 @@ Common issues: * No events captured: verify one enablement gate is set and your hook manifest is active. * No enrichment data: model and token enrichment depends on available debug logs and session-state data. -* Report generation fails: install `jq` and ensure `python3` is available. +* Report generation fails: ensure `python3` is available for enrichment. The bash generator also needs `jq`; the PowerShell generator (`Invoke-TelemetryReport.ps1`) does not. ## Related Guides From 73a7f672e9258a2ddee49fda7c1db25b68d0c998 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 18 Jun 2026 10:36:12 -0600 Subject: [PATCH 03/10] update hook events, update hook to be collection-scoped, escape chars in html, add opt-in for raw capture --- .github/hooks/shared/telemetry.json | 70 +++++++++ .../telemetry/Invoke-TelemetryClean.ps1 | 0 .../telemetry/Invoke-TelemetryCollector.ps1 | 0 .../telemetry/Invoke-TelemetryReport.ps1 | 0 .../{ => shared}/telemetry/_telemetry_core.py | 1 + .../{ => shared}/telemetry/clean-telemetry.sh | 0 .../telemetry/generate-telemetry-report.sh | 0 .../{ => shared}/telemetry/pyproject.toml | 0 .../hooks/{ => shared}/telemetry/report.html | 55 ++++--- .../telemetry/telemetry-collector.sh | 23 ++- .../telemetry/tests/fuzz_harness.py | 0 .../telemetry/tests/test_telemetry_core.py | 0 .github/hooks/{ => shared}/telemetry/uv.lock | 0 .github/hooks/telemetry.json | 142 ------------------ TRANSPARENCY-NOTE.md | 6 +- collections/hve-core-all.collection.yml | 2 +- collections/hve-core.collection.yml | 2 +- docs/contributing/hooks.md | 54 +++++-- docs/customization/local-telemetry.md | 91 ++++++++--- .../hve-core-all/.github/plugin/plugin.json | 2 +- plugins/hve-core-all/hooks/shared/telemetry | 1 + .../hve-core-all/hooks/shared/telemetry.json | 70 +++++++++ plugins/hve-core-all/hooks/telemetry | 1 - plugins/hve-core-all/hooks/telemetry.json | 142 ------------------ plugins/hve-core/.github/plugin/plugin.json | 2 +- plugins/hve-core/hooks/shared/telemetry | 1 + plugins/hve-core/hooks/shared/telemetry.json | 70 +++++++++ plugins/hve-core/hooks/telemetry | 1 - plugins/hve-core/hooks/telemetry.json | 142 ------------------ .../Modules/CollectionHelpers.psm1 | 44 +++--- scripts/extension/Prepare-Extension.ps1 | 5 +- scripts/plugins/Modules/PluginHelpers.psm1 | 2 +- .../collections/CollectionHelpers.Tests.ps1 | 41 +++++ 33 files changed, 443 insertions(+), 527 deletions(-) create mode 100644 .github/hooks/shared/telemetry.json rename .github/hooks/{ => shared}/telemetry/Invoke-TelemetryClean.ps1 (100%) rename .github/hooks/{ => shared}/telemetry/Invoke-TelemetryCollector.ps1 (100%) rename .github/hooks/{ => shared}/telemetry/Invoke-TelemetryReport.ps1 (100%) rename .github/hooks/{ => shared}/telemetry/_telemetry_core.py (99%) rename .github/hooks/{ => shared}/telemetry/clean-telemetry.sh (100%) rename .github/hooks/{ => shared}/telemetry/generate-telemetry-report.sh (100%) rename .github/hooks/{ => shared}/telemetry/pyproject.toml (100%) rename .github/hooks/{ => shared}/telemetry/report.html (94%) rename .github/hooks/{ => shared}/telemetry/telemetry-collector.sh (70%) rename .github/hooks/{ => shared}/telemetry/tests/fuzz_harness.py (100%) rename .github/hooks/{ => shared}/telemetry/tests/test_telemetry_core.py (100%) rename .github/hooks/{ => shared}/telemetry/uv.lock (100%) delete mode 100644 .github/hooks/telemetry.json create mode 120000 plugins/hve-core-all/hooks/shared/telemetry create mode 100644 plugins/hve-core-all/hooks/shared/telemetry.json delete mode 120000 plugins/hve-core-all/hooks/telemetry delete mode 100644 plugins/hve-core-all/hooks/telemetry.json create mode 120000 plugins/hve-core/hooks/shared/telemetry create mode 100644 plugins/hve-core/hooks/shared/telemetry.json delete mode 120000 plugins/hve-core/hooks/telemetry delete mode 100644 plugins/hve-core/hooks/telemetry.json diff --git a/.github/hooks/shared/telemetry.json b/.github/hooks/shared/telemetry.json new file mode 100644 index 000000000..d39599b38 --- /dev/null +++ b/.github/hooks/shared/telemetry.json @@ -0,0 +1,70 @@ +{ + "version": 1, + "description": "Records Copilot session lifecycle events to local telemetry for reporting.", + "hooks": { + "sessionStart": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "userPromptSubmit": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preToolUse": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "postToolUse": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStart": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStop": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "stop": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preCompact": [ + { + "type": "command", + "bash": ".github/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": ".github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ] + } +} diff --git a/.github/hooks/telemetry/Invoke-TelemetryClean.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 similarity index 100% rename from .github/hooks/telemetry/Invoke-TelemetryClean.ps1 rename to .github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 diff --git a/.github/hooks/telemetry/Invoke-TelemetryCollector.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 similarity index 100% rename from .github/hooks/telemetry/Invoke-TelemetryCollector.ps1 rename to .github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 diff --git a/.github/hooks/telemetry/Invoke-TelemetryReport.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 similarity index 100% rename from .github/hooks/telemetry/Invoke-TelemetryReport.ps1 rename to .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 diff --git a/.github/hooks/telemetry/_telemetry_core.py b/.github/hooks/shared/telemetry/_telemetry_core.py similarity index 99% rename from .github/hooks/telemetry/_telemetry_core.py rename to .github/hooks/shared/telemetry/_telemetry_core.py index 60c264c5b..b34cc34d0 100644 --- a/.github/hooks/telemetry/_telemetry_core.py +++ b/.github/hooks/shared/telemetry/_telemetry_core.py @@ -66,6 +66,7 @@ def iter_jsonl(path: str | os.PathLike[str]) -> Iterator[dict]: if isinstance(obj, dict): yield obj except OSError: + # File cannot be opened or read (e.g., does not exist or permission denied) return diff --git a/.github/hooks/telemetry/clean-telemetry.sh b/.github/hooks/shared/telemetry/clean-telemetry.sh similarity index 100% rename from .github/hooks/telemetry/clean-telemetry.sh rename to .github/hooks/shared/telemetry/clean-telemetry.sh diff --git a/.github/hooks/telemetry/generate-telemetry-report.sh b/.github/hooks/shared/telemetry/generate-telemetry-report.sh similarity index 100% rename from .github/hooks/telemetry/generate-telemetry-report.sh rename to .github/hooks/shared/telemetry/generate-telemetry-report.sh diff --git a/.github/hooks/telemetry/pyproject.toml b/.github/hooks/shared/telemetry/pyproject.toml similarity index 100% rename from .github/hooks/telemetry/pyproject.toml rename to .github/hooks/shared/telemetry/pyproject.toml diff --git a/.github/hooks/telemetry/report.html b/.github/hooks/shared/telemetry/report.html similarity index 94% rename from .github/hooks/telemetry/report.html rename to .github/hooks/shared/telemetry/report.html index 3fdd70fd0..c74df2d46 100644 --- a/.github/hooks/telemetry/report.html +++ b/.github/hooks/shared/telemetry/report.html @@ -153,6 +153,19 @@

Tool Latency

// Maps subagent tool_call_id → human-readable agent name (populated from SessionSummary) let subagentNames = {}; + /** Escape a value for safe interpolation into HTML text or attributes. + Session content (prompts, cwd, tool/instruction/skill/subagent names) can + originate from models, MCP servers, or third-party collections, so every + dynamic value written into innerHTML must pass through here. */ + function esc(v) { + return String(v == null ? '' : v) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + } + dropZone.addEventListener('click', () => fileInput.click()); dropZone.addEventListener('dragover', e => { e.preventDefault(); dropZone.classList.add('active'); }); dropZone.addEventListener('dragleave', () => dropZone.classList.remove('active')); @@ -450,7 +463,7 @@

Tool Latency

} function render() { - fileListEl.innerHTML = loadedFiles.map(f => `${f}`).join(' '); + fileListEl.innerHTML = loadedFiles.map(f => `${esc(f)}`).join(' '); dashboard.classList.add('visible'); const sessions = buildSessions(); renderOverview(sessions); @@ -499,7 +512,7 @@

Tool Latency

{ label: 'Output Tokens', value: fmtTokens(totalOut) } ); } - grid.innerHTML = cards.map(c => `
${c.label}
${c.value}
`).join(''); + grid.innerHTML = cards.map(c => `
${esc(c.label)}
${esc(c.value)}
`).join(''); } function renderMatrix(sessions) { @@ -509,12 +522,12 @@

Tool Latency

const dim = (none) => `${none}`; const cols = sessions.map(s => { const shortSid = s.sid.substring(0, 12); - const models = sessionModels(s).map(m => `${m}`).join('') || `not yet detected`; - const agents = [...s.subagents].filter(Boolean).map(a => `${a}`).join('') || dim('none'); - const instrs = [...s.instructions].map(i => `${i}`).join('') || dim('none loaded'); - const skills = [...s.skills].map(sk => `${sk}`).join('') || dim('none loaded'); + const models = sessionModels(s).map(m => `${esc(m)}`).join('') || `not yet detected`; + const agents = [...s.subagents].filter(Boolean).map(a => `${esc(a)}`).join('') || dim('none'); + const instrs = [...s.instructions].map(i => `${esc(i)}`).join('') || dim('none loaded'); + const skills = [...s.skills].map(sk => `${esc(sk)}`).join('') || dim('none loaded'); const topTools = Object.entries(s.tools).sort((a,b) => b[1] - a[1]).slice(0, 8) - .map(([t,c]) => `${t} (${c})`).join('') || dim('none'); + .map(([t,c]) => `${esc(t)} (${c})`).join('') || dim('none'); const durMs = new Date(s.lastTs) - new Date(s.firstTs); const durStr = durMs > 0 ? fmtDuration(durMs) : ''; @@ -534,7 +547,7 @@

Tool Latency

.sort((a, b) => (b[1].output_tokens || 0) - (a[1].output_tokens || 0)) .map(([name, au]) => { const freshIn = Math.max(0, (au.input_tokens || 0) - (au.cache_read_tokens || 0)); - return `
${name}` + + return `
${esc(name)}` + `${fmtTokens(freshIn)} in · ${fmtTokens(au.output_tokens || 0)} out` + (au.total_nano_aiu ? ` · ${fmtAiu(au.total_nano_aiu)} AIU` : '') + ` · ${au.requests || 0} req
`; @@ -545,12 +558,12 @@

Tool Latency

} const cwdShort = s.cwd ? s.cwd.replace(/^.*[/\\]/, '') : ''; - const clientTag = s.client ? `${s.client}` : ''; + const clientTag = s.client ? `${esc(s.client)}` : ''; return { - header: `${sessionLabel(s)}` + - `${shortSid}` + clientTag + - (cwdShort ? `${cwdShort}` : '') + + header: `${esc(sessionLabel(s))}` + + `${esc(shortSid)}` + clientTag + + (cwdShort ? `${esc(cwdShort)}` : '') + `${s.toolCount} tool calls` + (durStr ? `${durStr} duration` : '') + (started ? `${started}` : ''), @@ -594,14 +607,14 @@

Tool Latency

let html = `
`; // Header row html += '
Session
'; - tools.forEach(t => { html += `
${t.replace(/^(vscode_|manage_)/, '')}
`; }); + tools.forEach(t => { html += `
${esc(t.replace(/^(vscode_|manage_)/, ''))}
`; }); // Data rows sessions.forEach(s => { const models = sessionModels(s); const modelStr = models.length ? models.map(shortModel).join(', ') : '—'; - html += `
` + - `${sessionLabel(s, 18)}${modelStr}
`; + html += `
` + + `${esc(sessionLabel(s, 18))}${esc(modelStr)}
`; tools.forEach(t => { const count = s.tools[t] || 0; const heat = count === 0 ? 0 : count <= 3 ? 1 : count <= 10 ? 2 : 3; @@ -675,7 +688,7 @@

Tool Latency

const min = Math.min(...times); const max = Math.max(...times); const total = times.reduce((s,v) => s+v, 0); - html += `${tool}${times.length}${fmtDuration(avg)}${fmtDuration(min)}${fmtDuration(max)}${fmtDuration(total)}`; + html += `${esc(tool)}${times.length}${fmtDuration(avg)}${fmtDuration(min)}${fmtDuration(max)}${fmtDuration(total)}`; }); html += ''; container.innerHTML = html; @@ -708,8 +721,8 @@

Tool Latency

const hitPct = u.input ? Math.round((u.cached / u.input) * 100) : 0; const avgTtft = u.ttftCount ? fmtDuration(u.ttft / u.ttftCount) : '—'; const models = sessionModels(s).join(', ') || '—'; - const clientTag = s.client ? ` ${s.client}` : ''; - html += `${sessionLabel(s, 40)}${clientTag}${models}` + + const clientTag = s.client ? ` ${esc(s.client)}` : ''; + html += `${esc(sessionLabel(s, 40))}${clientTag}${esc(models)}` + `${u.requests}${inputCell(u)}` + `${fmtTokens(u.cached)} (${hitPct}%)${fmtTokens(u.output)}` + `${fmtTokens(total)}${u.aiu ? fmtAiu(u.aiu) : '—'}${avgTtft}`; @@ -718,7 +731,7 @@

Tool Latency

if (mu && Object.keys(mu).length > 1) { Object.entries(mu).sort((a, b) => b[1].output_tokens - a[1].output_tokens).forEach(([m, d]) => { const pct = u.output ? Math.round((d.output_tokens / u.output) * 100) : 0; - html += `↳ ${m}` + + html += `↳ ${esc(m)}` + `${d.messages}` + `${fmtTokens(d.output_tokens)} (${pct}%)`; }); @@ -779,12 +792,12 @@

Tool Latency

let html = `
`; // Header row html += '
Artifact
'; - sessions.forEach(s => { html += `
${sessionLabel(s, 18)}
`; }); + sessions.forEach(s => { html += `
${esc(sessionLabel(s, 18))}
`; }); // Artifact rows rows.forEach(r => { const tagClass = r.kind === 'instr' ? 'tag-instruction' : 'tag-skill'; - html += `
${r.label}
`; + html += `
${esc(r.label)}
`; sessions.forEach(s => { const loaded = r.kind === 'instr' ? s.instructions.has(r.name) : s.skills.has(r.name); html += cell(tokensFor(s, r), loaded); diff --git a/.github/hooks/telemetry/telemetry-collector.sh b/.github/hooks/shared/telemetry/telemetry-collector.sh similarity index 70% rename from .github/hooks/telemetry/telemetry-collector.sh rename to .github/hooks/shared/telemetry/telemetry-collector.sh index c35abde60..de833dd38 100755 --- a/.github/hooks/telemetry/telemetry-collector.sh +++ b/.github/hooks/shared/telemetry/telemetry-collector.sh @@ -41,14 +41,21 @@ main() { local telemetry_dir="${HVE_TELEMETRY_DIR:-$repo_root/.copilot-tracking/telemetry}" mkdir -p "$telemetry_dir" "$telemetry_dir/.stacks" - # Dump raw input for diagnostics (first 5 events only) - local raw_log="$telemetry_dir/raw-input.jsonl" - local raw_count=0 - if [[ -f "$raw_log" ]]; then - raw_count=$(wc -l < "$raw_log") - fi - if (( raw_count < 5 )); then - echo "$input" >> "$raw_log" + # Dump raw input for diagnostics (first 5 events only). This records hook + # payloads verbatim, including the full prompt text and tool inputs such as + # file contents and shell command strings, which can contain secrets. The + # processed sessions-*.jsonl stream already provides the diagnostic signal, + # so the verbatim dump is a separate explicit opt-in (off by default) layered + # on top of the telemetry gate. See docs/customization/local-telemetry.md. + if [[ "${HVE_TELEMETRY_RAW:-}" == "1" ]]; then + local raw_log="$telemetry_dir/raw-input.jsonl" + local raw_count=0 + if [[ -f "$raw_log" ]]; then + raw_count=$(wc -l < "$raw_log") + fi + if (( raw_count < 5 )); then + echo "$input" >> "$raw_log" + fi fi # Delegate all JSON processing to the shared telemetry engine. The engine diff --git a/.github/hooks/telemetry/tests/fuzz_harness.py b/.github/hooks/shared/telemetry/tests/fuzz_harness.py similarity index 100% rename from .github/hooks/telemetry/tests/fuzz_harness.py rename to .github/hooks/shared/telemetry/tests/fuzz_harness.py diff --git a/.github/hooks/telemetry/tests/test_telemetry_core.py b/.github/hooks/shared/telemetry/tests/test_telemetry_core.py similarity index 100% rename from .github/hooks/telemetry/tests/test_telemetry_core.py rename to .github/hooks/shared/telemetry/tests/test_telemetry_core.py diff --git a/.github/hooks/telemetry/uv.lock b/.github/hooks/shared/telemetry/uv.lock similarity index 100% rename from .github/hooks/telemetry/uv.lock rename to .github/hooks/shared/telemetry/uv.lock diff --git a/.github/hooks/telemetry.json b/.github/hooks/telemetry.json deleted file mode 100644 index 74a205e3f..000000000 --- a/.github/hooks/telemetry.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "version": 1, - "description": "Records Copilot session lifecycle events to local telemetry for reporting.", - "hooks": { - "sessionStart": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SessionStart": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "userPromptSubmitted": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "UserPromptSubmit": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "preToolUse": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PreToolUse": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "postToolUse": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PostToolUse": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "subagentStart": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SubagentStart": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "subagentStop": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SubagentStop": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "agentStop": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "Stop": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "sessionEnd": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "preCompact": [ - { - "type": "command", - "bash": ".github/hooks/telemetry/telemetry-collector.sh", - "powershell": ".github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PreCompact": [ - { - "type": "command", - "command": ".github/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File .github/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ] - } -} diff --git a/TRANSPARENCY-NOTE.md b/TRANSPARENCY-NOTE.md index e698c1437..05233c10b 100644 --- a/TRANSPARENCY-NOTE.md +++ b/TRANSPARENCY-NOTE.md @@ -2,7 +2,7 @@ title: "Transparency Note: HVE Core (May 2026)" description: "Public Transparency Note for HVE Core, a prompt-engineering and agentic-customization framework distributed by microsoft/hve-core." author: HVE Core Maintainers -ms.date: 2026-06-11 +ms.date: 2026-06-17 ms.topic: overview keywords: - responsible-ai @@ -56,7 +56,9 @@ HVE Core ships text files and supporting tools. When you load an HVE Core file i 2. You work with the host's model, now shaped by the file's instructions. 3. The model's replies come back through the normal Copilot surface. -HVE Core has no model, no API, no network calls while you author or install it, and no telemetry. Validation tools (linters, frontmatter checks, Pester tests, plugin generation) run in CI on pull requests. Nothing runs on your machine unless you install the VS Code extension or run a packaged script yourself. +HVE Core has no model, no API, and no network calls while you author or install it. It ships one optional local telemetry hook that is disabled by default and, when you turn it on, records Copilot session lifecycle events to plaintext files on your own disk with no network egress. +The processed event stream stores derived signals (such as tool-input key names and a truncated prompt preview) rather than full payloads; a separate, explicit opt-in is required before any verbatim prompt or tool input is captured. See the [Local Telemetry guide](docs/customization/local-telemetry.md) for exactly what is captured and how to disable or remove it. +Validation tools (linters, frontmatter checks, Pester tests, plugin generation) run in CI on pull requests. Nothing runs on your machine unless you install the VS Code extension or run a packaged script yourself. Most skills are pure authoring or validation helpers with no independent Responsible AI surface and are not called out individually. A few skills warrant specific mention because they assemble media outputs or depend on external services: diff --git a/collections/hve-core-all.collection.yml b/collections/hve-core-all.collection.yml index 5056b3f89..169c4bc9a 100644 --- a/collections/hve-core-all.collection.yml +++ b/collections/hve-core-all.collection.yml @@ -599,7 +599,7 @@ items: kind: skill - path: .github/skills/shared/telemetry-foundations kind: skill -- path: .github/hooks/telemetry.json +- path: .github/hooks/shared/telemetry.json kind: hook display: featured: true diff --git a/collections/hve-core.collection.yml b/collections/hve-core.collection.yml index 4b7114e22..6a4d6765d 100644 --- a/collections/hve-core.collection.yml +++ b/collections/hve-core.collection.yml @@ -150,7 +150,7 @@ items: - path: .github/instructions/shared/telemetry-overlay.instructions.md kind: instruction # Hooks - - path: .github/hooks/telemetry.json + - path: .github/hooks/shared/telemetry.json kind: hook display: ordering: manual diff --git a/docs/contributing/hooks.md b/docs/contributing/hooks.md index e2904a175..25740c5ac 100644 --- a/docs/contributing/hooks.md +++ b/docs/contributing/hooks.md @@ -3,7 +3,7 @@ title: Contributing Hooks description: How to implement, register, and validate hook artifacts in hve-core sidebar_position: 7 author: Microsoft -ms.date: 2026-06-08 +ms.date: 2026-06-17 ms.topic: how-to keywords: - hooks @@ -25,24 +25,26 @@ Use a hook when you need event-driven behavior such as: ## Hook Layout in This Repository -Use this structure for hook contributions: +Hooks are collection-scoped, like every other distributable artifact type. Use this structure for hook contributions: | Path | Purpose | |---|---| -| `.github/hooks/.json` | Hook manifest that maps lifecycle events to executable commands | -| `.github/hooks//` | Hook implementation scripts and support files | +| `.github/hooks//.json` | Hook manifest that maps lifecycle events to executable commands | +| `.github/hooks///` | Hook implementation scripts and support files | | `collections/*.collection.yml` | Collection registration with `kind: hook` | | `collections/*.collection.md` | Human-readable hook entry in the collection documentation table | +Manifests live one collection level down (`.github/hooks//`) so the installer can activate each collection's hooks independently by adding only that collection's folder to `chat.hookFilesLocations`. A flat `.github/hooks/.json` is treated as a repo-specific artifact and is excluded from distribution. + The telemetry hook is the current reference implementation: -* `.github/hooks/telemetry.json` -* `.github/hooks/telemetry/` +* `.github/hooks/shared/telemetry.json` +* `.github/hooks/shared/telemetry/` ## Implementing a New Hook -1. Add a manifest at `.github/hooks/.json`. -2. Add executable scripts under `.github/hooks//`. +1. Add a manifest at `.github/hooks//.json`. +2. Add executable scripts under `.github/hooks///`. 3. Register the hook in one or more `collections/*.collection.yml` files. 4. Document the hook in the matching `collections/*.collection.md` files. 5. Add or update docs under `docs/` for setup and usage. @@ -56,8 +58,8 @@ Minimal manifest pattern: "preToolUse": [ { "type": "command", - "bash": ".github/hooks/my-hook/my-hook.sh", - "powershell": ".github/hooks/my-hook/Invoke-MyHook.ps1", + "bash": ".github/hooks/shared/my-hook/my-hook.sh", + "powershell": ".github/hooks/shared/my-hook/Invoke-MyHook.ps1", "timeoutSec": 10 } ] @@ -78,15 +80,35 @@ For reliability and portability, hook scripts should follow these rules: Telemetry follows this model with a no-op gate and structured JSONL append behavior. +## Handling Sensitive Payloads + +Hook payloads can contain sensitive data. `PreToolUse` inputs include full file +contents being written and shell command strings, and `UserPromptSubmit` +includes the full prompt, any of which may carry secrets. Follow these rules +when a hook persists payloads to disk: + +* Store only the minimum needed for the hook's purpose. Prefer derived signals + (keys, lengths, counts, truncated previews) over verbatim values. +* Gate any verbatim payload capture behind its own explicit opt-in, separate + from the hook's main enable gate, and default it off. +* Write to local, gitignored locations and never to committed paths. +* Document exactly what is captured, where it is written, and how to remove it. + +The telemetry hook applies this pattern: its processed `sessions-*.jsonl` stream +stores only tool-input key names and a truncated prompt preview, while the +verbatim `raw-input.jsonl` dump is a separate opt-in (`HVE_TELEMETRY_RAW=1`, +off by default). See [Local Telemetry](../customization/local-telemetry#sensitive-data-and-privacy). + ## Event Compatibility Guidance -The telemetry manifest includes both lowercase and PascalCase event names to support multiple invocation surfaces. If you need broad compatibility across environments, mirror that pattern in your hook manifest. +Write a single CLI-format block per event: lowercase event keys with `bash` and `powershell` command properties. VS Code automatically converts the lowercase CLI event names to its PascalCase form and maps `bash` to `osx`/`linux` and `powershell` to `windows`, so one block covers both surfaces. Do not also declare a PascalCase copy of the same event; VS Code would register and fire both, duplicating every invocation. -Examples from telemetry: +Choose CLI event names that convert to valid VS Code events: -* `sessionStart` and `SessionStart` -* `preToolUse` and `PreToolUse` -* `agentStop` and `Stop` +* `sessionStart` -> `SessionStart` +* `preToolUse` -> `PreToolUse` +* `userPromptSubmit` -> `UserPromptSubmit` (not `userPromptSubmitted`) +* `stop` -> `Stop` (VS Code has no `sessionEnd` or `agentStop` event) ## Registering a Hook in Collections @@ -94,7 +116,7 @@ Add a collection item with `kind: hook`: ```yaml items: - - path: .github/hooks/my-hook.json + - path: .github/hooks//my-hook.json kind: hook ``` diff --git a/docs/customization/local-telemetry.md b/docs/customization/local-telemetry.md index 7e6c34fd8..b806b21e0 100644 --- a/docs/customization/local-telemetry.md +++ b/docs/customization/local-telemetry.md @@ -3,7 +3,7 @@ title: Local Telemetry description: Enable local Copilot session telemetry, understand capture mechanics, and generate local reports sidebar_position: 10 author: Microsoft -ms.date: 2026-06-08 +ms.date: 2026-06-17 ms.topic: how-to keywords: - telemetry @@ -17,7 +17,7 @@ estimated_reading_time: 7 The local telemetry hook captures Copilot lifecycle events into local JSONL files. It is intended for local analysis and troubleshooting of your own sessions. -The telemetry manifest is at `.github/hooks/telemetry.json`. +The telemetry manifest is at `.github/hooks/shared/telemetry.json`. Events currently captured include: @@ -54,6 +54,22 @@ touch .hve-telemetry Either option enables collection. If both are absent, the hook exits in no-op mode. +### Optional: Verbatim Raw Payload Capture + +Processed telemetry never stores full prompt text or full tool inputs (see +[Sensitive Data and Privacy](#sensitive-data-and-privacy)). A separate, +explicit opt-in records the first few hook payloads **verbatim** to +`raw-input.jsonl` for deep diagnostics. It is off by default, even when +telemetry is enabled, and is honored only by the Bash collector: + +```bash +export HVE_TELEMETRY_RAW=1 +``` + +Leave this unset unless you are actively debugging the hook payload shape, and +remove the captured file afterward. Because it stores prompts and tool inputs in +the clear, treat any session run with it enabled as potentially sensitive. + ## View Reports Generate a report with the script in ~/.hve (created at session start): @@ -89,12 +105,12 @@ Override with `HVE_TELEMETRY_DIR` when needed. Key files and folders: -| Path | Purpose | -|---|---| -| `sessions-YYYY-MM-DD.jsonl` | Daily event stream with hook events and session summaries | -| `raw-input.jsonl` | First few raw hook payloads for diagnostics | -| `.stacks/` | Per-session agent stack tracking used for attribution | -| `report.generated.html` | Optional self-contained report output | +| Path | Purpose | +|-----------------------------|----------------------------------------------------------------------------------------------------------| +| `sessions-YYYY-MM-DD.jsonl` | Daily event stream with hook events and session summaries | +| `raw-input.jsonl` | First few hook payloads stored verbatim; written only when `HVE_TELEMETRY_RAW=1` is set (Bash collector) | +| `.stacks/` | Per-session agent stack tracking used for attribution | +| `report.generated.html` | Optional self-contained report output | ## Data Captured and Storage Schema @@ -102,7 +118,7 @@ This section describes the mechanics of what local telemetry collects and where ### Collection Pipeline -1. Copilot lifecycle events invoke the telemetry hook from `.github/hooks/telemetry.json`. +1. Copilot lifecycle events invoke the telemetry hook from `.github/hooks/shared/telemetry.json`. 2. Shell entry points (`telemetry-collector.sh` and `Invoke-TelemetryCollector.ps1`) enforce opt-in gates. 3. Event payloads are normalized and appended to daily JSONL files. 4. On stop events, a `SessionSummary` record is appended with model and token aggregates when available. @@ -111,10 +127,10 @@ This section describes the mechanics of what local telemetry collects and where The daily JSONL stream contains two primary record types: -| Record Type | Trigger | Purpose | -|---|---|---| -| Hook event records | Session/tool/subagent lifecycle events | Timeline of what happened during a session | -| `SessionSummary` | Stop event (`Stop`) | Aggregated usage totals and model-level summary | +| Record Type | Trigger | Purpose | +|--------------------|----------------------------------------|-------------------------------------------------| +| Hook event records | Session/tool/subagent lifecycle events | Timeline of what happened during a session | +| `SessionSummary` | Stop event (`Stop`) | Aggregated usage totals and model-level summary | ### Common Fields @@ -146,10 +162,10 @@ When available at stop time, `SessionSummary` includes: ### Data Sources by Layer -| Data Category | Source | -|---|---| -| Hook lifecycle events | Copilot hook payloads routed through collector scripts | -| Session summaries | `.copilot/session-state//events.jsonl` (CLI session state) | +| Data Category | Source | +|-----------------------------------------------|---------------------------------------------------------------------------| +| Hook lifecycle events | Copilot hook payloads routed through collector scripts | +| Session summaries | `.copilot/session-state//events.jsonl` (CLI session state) | | Additional model/token enrichment for reports | VS Code debug logs and session-state aggregation during report generation | ### Event Naming Normalization @@ -160,24 +176,49 @@ The pipeline normalizes different casing variants of event names to canonical na * Data is stored locally under `.copilot-tracking/telemetry` by default. * Records append to date-partitioned files (`sessions-YYYY-MM-DD.jsonl`). -* A small raw payload diagnostic sample is stored in `raw-input.jsonl`. +* A small verbatim raw payload sample is stored in `raw-input.jsonl` only when `HVE_TELEMETRY_RAW=1` is explicitly set; see [Sensitive Data and Privacy](#sensitive-data-and-privacy). * Per-session agent stack files are maintained under `.stacks/` for attribution and cleaned up on session stop. +### Sensitive Data and Privacy + +Local telemetry writes plaintext JSONL to local disk only. It makes no network +calls and the default output directory (`.copilot-tracking/telemetry`) is +gitignored, so data is not committed. The risk is local-disk exposure, not a +committed leak. Be aware of what each layer records: + +* **Processed stream (`sessions-*.jsonl`)** stores a truncated prompt preview + (first 200 characters of each submitted prompt) and, for tool events, only + the tool input *key names* plus selected fields such as file paths and + subagent names. It does not store full tool input *values* (file contents or + shell command strings). A secret pasted into the start of a prompt can still + appear in the 200-character preview. +* **Verbatim raw dump (`raw-input.jsonl`)** stores the first few hook payloads + exactly as received, including the full prompt and the full tool input (file + contents being written, shell command strings). It is off by default and only + written when `HVE_TELEMETRY_RAW=1` is set. +* **User-level locations** under `~/.hve` and `~/.copilot` (honoring `HVE_HOME`) + hold the report generator and directory registry. Generated reports embed the + captured JSONL inline. + +To reduce exposure: keep `HVE_TELEMETRY_RAW` unset, avoid pasting secrets into +prompts while telemetry is enabled, and remove captured files when you are done +(`bash ~/.hve/clean-telemetry.sh` or delete the telemetry directory). + ## Generate a Report Run the report generator directly: ```bash -bash .github/hooks/telemetry/generate-telemetry-report.sh --help -bash .github/hooks/telemetry/generate-telemetry-report.sh --date all -bash .github/hooks/telemetry/generate-telemetry-report.sh --open +bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --help +bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --date all +bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --open ``` On Windows (or any PowerShell host) the native equivalent needs no `bash`: ```powershell -pwsh .github/hooks/telemetry/Invoke-TelemetryReport.ps1 -Date all -pwsh .github/hooks/telemetry/Invoke-TelemetryReport.ps1 -Open +pwsh .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 -Date all +pwsh .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 -Open ``` ## Cross-Project Reports @@ -190,13 +231,13 @@ at `~/.hve/telemetry-dirs.txt` (honoring `HVE_HOME`). Generate a combined, cross-project report with `--all-dirs`: ```bash -bash .github/hooks/telemetry/generate-telemetry-report.sh --all-dirs --date all +bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --all-dirs --date all ``` The PowerShell generator takes `-AllDirs` for the same cross-project report: ```powershell -pwsh .github/hooks/telemetry/Invoke-TelemetryReport.ps1 -AllDirs -Date all +pwsh .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 -AllDirs -Date all ``` The registry self-populates as you work across repositories, so no manual setup diff --git a/plugins/hve-core-all/.github/plugin/plugin.json b/plugins/hve-core-all/.github/plugin/plugin.json index 2355c5deb..5117d3835 100644 --- a/plugins/hve-core-all/.github/plugin/plugin.json +++ b/plugins/hve-core-all/.github/plugin/plugin.json @@ -50,5 +50,5 @@ "skills/security/", "skills/shared/" ], - "hooks": "hooks/telemetry.json" + "hooks": "hooks/shared/telemetry.json" } \ No newline at end of file diff --git a/plugins/hve-core-all/hooks/shared/telemetry b/plugins/hve-core-all/hooks/shared/telemetry new file mode 120000 index 000000000..88b5d21b1 --- /dev/null +++ b/plugins/hve-core-all/hooks/shared/telemetry @@ -0,0 +1 @@ +../../../../.github/hooks/shared/telemetry \ No newline at end of file diff --git a/plugins/hve-core-all/hooks/shared/telemetry.json b/plugins/hve-core-all/hooks/shared/telemetry.json new file mode 100644 index 000000000..ed7dd2eef --- /dev/null +++ b/plugins/hve-core-all/hooks/shared/telemetry.json @@ -0,0 +1,70 @@ +{ + "version": 1, + "description": "Records Copilot session lifecycle events to local telemetry for reporting.", + "hooks": { + "sessionStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "userPromptSubmit": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "postToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "stop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preCompact": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ] + } +} diff --git a/plugins/hve-core-all/hooks/telemetry b/plugins/hve-core-all/hooks/telemetry deleted file mode 120000 index 52d07b1e3..000000000 --- a/plugins/hve-core-all/hooks/telemetry +++ /dev/null @@ -1 +0,0 @@ -../../../.github/hooks/telemetry \ No newline at end of file diff --git a/plugins/hve-core-all/hooks/telemetry.json b/plugins/hve-core-all/hooks/telemetry.json deleted file mode 100644 index 285749377..000000000 --- a/plugins/hve-core-all/hooks/telemetry.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "version": 1, - "description": "Records Copilot session lifecycle events to local telemetry for reporting.", - "hooks": { - "sessionStart": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SessionStart": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "userPromptSubmitted": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "UserPromptSubmit": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "preToolUse": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PreToolUse": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "postToolUse": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PostToolUse": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "subagentStart": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SubagentStart": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "subagentStop": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SubagentStop": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "agentStop": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "Stop": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "sessionEnd": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "preCompact": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PreCompact": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ] - } -} diff --git a/plugins/hve-core/.github/plugin/plugin.json b/plugins/hve-core/.github/plugin/plugin.json index f5efc341d..3cf1b0655 100644 --- a/plugins/hve-core/.github/plugin/plugin.json +++ b/plugins/hve-core/.github/plugin/plugin.json @@ -14,5 +14,5 @@ "skills/hve-core/", "skills/shared/" ], - "hooks": "hooks/telemetry.json" + "hooks": "hooks/shared/telemetry.json" } \ No newline at end of file diff --git a/plugins/hve-core/hooks/shared/telemetry b/plugins/hve-core/hooks/shared/telemetry new file mode 120000 index 000000000..88b5d21b1 --- /dev/null +++ b/plugins/hve-core/hooks/shared/telemetry @@ -0,0 +1 @@ +../../../../.github/hooks/shared/telemetry \ No newline at end of file diff --git a/plugins/hve-core/hooks/shared/telemetry.json b/plugins/hve-core/hooks/shared/telemetry.json new file mode 100644 index 000000000..ed7dd2eef --- /dev/null +++ b/plugins/hve-core/hooks/shared/telemetry.json @@ -0,0 +1,70 @@ +{ + "version": 1, + "description": "Records Copilot session lifecycle events to local telemetry for reporting.", + "hooks": { + "sessionStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "userPromptSubmit": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "postToolUse": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStart": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "subagentStop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "stop": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ], + "preCompact": [ + { + "type": "command", + "bash": "${PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh", + "powershell": "${PLUGIN_ROOT}/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1", + "timeoutSec": 10 + } + ] + } +} diff --git a/plugins/hve-core/hooks/telemetry b/plugins/hve-core/hooks/telemetry deleted file mode 120000 index 52d07b1e3..000000000 --- a/plugins/hve-core/hooks/telemetry +++ /dev/null @@ -1 +0,0 @@ -../../../.github/hooks/telemetry \ No newline at end of file diff --git a/plugins/hve-core/hooks/telemetry.json b/plugins/hve-core/hooks/telemetry.json deleted file mode 100644 index 285749377..000000000 --- a/plugins/hve-core/hooks/telemetry.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "version": 1, - "description": "Records Copilot session lifecycle events to local telemetry for reporting.", - "hooks": { - "sessionStart": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SessionStart": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "userPromptSubmitted": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "UserPromptSubmit": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "preToolUse": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PreToolUse": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "postToolUse": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PostToolUse": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "subagentStart": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SubagentStart": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "subagentStop": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "SubagentStop": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "agentStop": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "Stop": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "sessionEnd": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "preCompact": [ - { - "type": "command", - "bash": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "powershell": "${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ], - "PreCompact": [ - { - "type": "command", - "command": "${PLUGIN_ROOT}/hooks/telemetry/telemetry-collector.sh", - "windows": "powershell -File ${PLUGIN_ROOT}/hooks/telemetry/Invoke-TelemetryCollector.ps1", - "timeoutSec": 10 - } - ] - } -} diff --git a/scripts/collections/Modules/CollectionHelpers.psm1 b/scripts/collections/Modules/CollectionHelpers.psm1 index 340c2681f..dde43962e 100644 --- a/scripts/collections/Modules/CollectionHelpers.psm1 +++ b/scripts/collections/Modules/CollectionHelpers.psm1 @@ -123,7 +123,7 @@ function Test-HveCoreRepoRelativePath { .DESCRIPTION Returns true when the repo-relative path is directly under a .github type - directory (agents, instructions, prompts, skills) with no subdirectory, + directory (agents, instructions, prompts, hooks, skills) with no subdirectory, indicating it is a root-level repo-specific artifact not intended for distribution. .PARAMETER Path @@ -140,7 +140,7 @@ function Test-HveCoreRepoRelativePath { [string]$Path ) - return ($Path -match '^\.github/(agents|instructions|prompts|skills)/[^/]+$') + return ($Path -match '^\.github/(agents|instructions|prompts|hooks|skills)/[^/]+$') } function Get-CollectionManifest { @@ -430,21 +430,28 @@ function Get-ArtifactFiles { } } - # Hooks (JSON files under .github/hooks/) + # Hooks (JSON manifests under .github/hooks//) $hooksDir = Join-Path -Path $RepoRoot -ChildPath '.github/hooks' if (Test-Path -Path $hooksDir) { - # Hook manifests live at the top level (.github/hooks/.json); - # implementation files under .github/hooks// must not be treated - # as manifests, so do not recurse. - $hookFiles = Get-ChildItem -Path $hooksDir -Filter '*.json' -File - foreach ($hookFile in $hookFiles) { - $relativePath = [System.IO.Path]::GetRelativePath($RepoRoot, $hookFile.FullName) -replace '\\', '/' - - if (Test-DeprecatedPath -Path $relativePath) { - continue + # Hook manifests are collection-scoped (.github/hooks//.json); + # implementation files live one level deeper under + # .github/hooks/// and must not be treated as + # manifests, so enumerate only the collection level. + $hookCollectionDirs = Get-ChildItem -Path $hooksDir -Directory + foreach ($collectionDir in $hookCollectionDirs) { + $hookFiles = Get-ChildItem -Path $collectionDir.FullName -Filter '*.json' -File + foreach ($hookFile in $hookFiles) { + $relativePath = [System.IO.Path]::GetRelativePath($RepoRoot, $hookFile.FullName) -replace '\\', '/' + + if (Test-HveCoreRepoRelativePath -Path $relativePath) { + continue + } + if (Test-DeprecatedPath -Path $relativePath) { + continue + } + + $items += @{ path = $relativePath; kind = 'hook' } } - - $items += @{ path = $relativePath; kind = 'hook' } } } @@ -725,13 +732,8 @@ function Get-ArtifactDescription { # description field instead of scanning for a YAML block. if ([System.IO.Path]::GetExtension($FilePath) -eq '.json') { try { - # Hook manifests can contain keys differing only by case (e.g. - # sessionStart / SessionStart), which ConvertFrom-Json rejects when - # building a case-insensitive PSCustomObject. -AsHashtable uses a - # case-sensitive map that tolerates those keys; only the top-level - # description is read here. - $json = Get-Content -Path $FilePath -Raw | ConvertFrom-Json -AsHashtable - $desc = $json['description'] + $json = Get-Content -Path $FilePath -Raw | ConvertFrom-Json + $desc = $json.description if ($desc) { return ([string]$desc).Trim() } diff --git a/scripts/extension/Prepare-Extension.ps1 b/scripts/extension/Prepare-Extension.ps1 index da22e42ce..d698321b7 100644 --- a/scripts/extension/Prepare-Extension.ps1 +++ b/scripts/extension/Prepare-Extension.ps1 @@ -411,6 +411,7 @@ function New-CollectionReadme { $prompts = @() $instructions = @() $skills = @() + $hooks = @() if ($Collection.ContainsKey('items')) { foreach ($item in $Collection.items) { @@ -438,6 +439,7 @@ function New-CollectionReadme { 'prompt' { $prompts += $entry } 'instruction' { $instructions += $entry } 'skill' { $skills += $entry } + 'hook' { $hooks += $entry } } } } @@ -449,7 +451,8 @@ function New-CollectionReadme { @{ Title = 'Chat Agents'; Items = $agents }, @{ Title = 'Prompts'; Items = $prompts }, @{ Title = 'Instructions'; Items = $instructions }, - @{ Title = 'Skills'; Items = $skills } + @{ Title = 'Skills'; Items = $skills }, + @{ Title = 'Hooks'; Items = $hooks } )) { if ($section.Items.Count -eq 0) { continue } diff --git a/scripts/plugins/Modules/PluginHelpers.psm1 b/scripts/plugins/Modules/PluginHelpers.psm1 index 56be660c6..23225b537 100644 --- a/scripts/plugins/Modules/PluginHelpers.psm1 +++ b/scripts/plugins/Modules/PluginHelpers.psm1 @@ -664,7 +664,7 @@ function Write-PluginHookArtifact { .DESCRIPTION Hook command paths in the source manifest are repository-root relative - (for example .github/hooks/telemetry/telemetry-collector.sh) so they resolve + (for example .github/hooks/shared/telemetry/telemetry-collector.sh) so they resolve when the hook is auto-loaded from a checked-out repository. Inside an installed plugin the same scripts live under the plugin root, so this function writes a transformed copy of the manifest with those paths diff --git a/scripts/tests/collections/CollectionHelpers.Tests.ps1 b/scripts/tests/collections/CollectionHelpers.Tests.ps1 index 46eb9978f..d703d9289 100644 --- a/scripts/tests/collections/CollectionHelpers.Tests.ps1 +++ b/scripts/tests/collections/CollectionHelpers.Tests.ps1 @@ -40,6 +40,20 @@ Describe 'Get-ArtifactFiles - repo-specific path exclusion' { $hveCorePromptsDir = Join-Path $promptsDir 'hve-core' New-Item -ItemType Directory -Path $hveCorePromptsDir -Force | Out-Null Set-Content -Path (Join-Path $hveCorePromptsDir 'task-plan.prompt.md') -Value '---\ndescription: distributable prompt\n---' + + # Create collection-scoped hook manifest in subdirectory (should be included) + $sharedHooksDir = Join-Path $ghDir 'hooks/shared' + New-Item -ItemType Directory -Path $sharedHooksDir -Force | Out-Null + Set-Content -Path (Join-Path $sharedHooksDir 'telemetry.json') -Value '{ "version": 1 }' + + # Create hook implementation file one level deeper (should NOT be treated as a manifest) + $hookImplDir = Join-Path $sharedHooksDir 'telemetry' + New-Item -ItemType Directory -Path $hookImplDir -Force | Out-Null + Set-Content -Path (Join-Path $hookImplDir 'config.json') -Value '{ "internal": true }' + + # Create flat hook manifest directly under .github/hooks (should be excluded) + $hooksRootDir = Join-Path $ghDir 'hooks' + Set-Content -Path (Join-Path $hooksRootDir 'orphan.json') -Value '{ "version": 1 }' } It 'Excludes root-level repo-specific instructions' { @@ -77,6 +91,25 @@ Describe 'Get-ArtifactFiles - repo-specific path exclusion' { $paths = $items | ForEach-Object { $_.path } $paths | Should -Contain '.github/prompts/hve-core/task-plan.prompt.md' } + + It 'Includes collection-scoped hook manifests in subdirectories' { + $items = Get-ArtifactFiles -RepoRoot $script:repoRoot + $hookItem = $items | Where-Object { $_.path -eq '.github/hooks/shared/telemetry.json' } + $hookItem | Should -Not -BeNullOrEmpty + $hookItem.kind | Should -Be 'hook' + } + + It 'Excludes hook implementation files nested under the manifest folder' { + $items = Get-ArtifactFiles -RepoRoot $script:repoRoot + $paths = $items | ForEach-Object { $_.path } + $paths | Should -Not -Contain '.github/hooks/shared/telemetry/config.json' + } + + It 'Excludes flat hook manifests directly under .github/hooks' { + $items = Get-ArtifactFiles -RepoRoot $script:repoRoot + $paths = $items | ForEach-Object { $_.path } + $paths | Should -Not -Contain '.github/hooks/orphan.json' + } } Describe 'Get-ArtifactFiles - deprecated path exclusion' { @@ -220,6 +253,14 @@ Describe 'Test-HveCoreRepoRelativePath' { Test-HveCoreRepoRelativePath -Path '.github/instructions/shared/hve-core-location.instructions.md' | Should -BeFalse } + It 'Returns true for flat hook manifest directly under hooks' { + Test-HveCoreRepoRelativePath -Path '.github/hooks/telemetry.json' | Should -BeTrue + } + + It 'Returns false for collection-scoped hook manifest' { + Test-HveCoreRepoRelativePath -Path '.github/hooks/shared/telemetry.json' | Should -BeFalse + } + It 'Returns false for path directly under .github (wrong nesting level)' { Test-HveCoreRepoRelativePath -Path '.github/foo.md' | Should -BeFalse } From 9288d303374fccc82138b307fe9ab783f26f843a Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 18 Jun 2026 15:26:08 -0600 Subject: [PATCH 04/10] add schema validation for hooks --- .github/PULL_REQUEST_TEMPLATE.md | 1 + .github/copilot-instructions.md | 2 + .../installer/hve-core-installer/SKILL.md | 13 +- .github/workflows/plugin-validation.yml | 3 + docs/contributing/hooks.md | 28 +- package.json | 3 +- scripts/linting/Validate-HookManifests.ps1 | 363 ++++++++++++++++++ .../linting/schemas/hook-manifest.schema.json | 144 +++++++ .../linting/Validate-HookManifests.Tests.ps1 | 189 +++++++++ scripts/tests/plugins/PluginHelpers.Tests.ps1 | 51 +++ 10 files changed, 785 insertions(+), 12 deletions(-) create mode 100644 scripts/linting/Validate-HookManifests.ps1 create mode 100644 scripts/linting/schemas/hook-manifest.schema.json create mode 100644 scripts/tests/linting/Validate-HookManifests.Tests.ps1 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4aa2b0354..072b18c4f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -32,6 +32,7 @@ Select all that apply: * [ ] Copilot prompt (`.github/prompts/*.prompt.md`) * [ ] Copilot agent (`.github/agents/*.agent.md`) * [ ] Copilot skill (`.github/skills/*/SKILL.md`) +* [ ] Copilot hook (`.github/hooks/*/*.json`) * [ ] Eval spec added/updated for changed AI artifacts (`evals/`) > Note for AI Artifact Contributors: diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d2aef48c6..659c3e57f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -51,6 +51,7 @@ The project is organized into these main areas: * Documentation (`docs/`) - Getting started guides, templates, RPI workflow documentation, and contribution guidelines. * Scripts (`scripts/`) - Automation for linting, security validation, extension packaging, and development tools. * Skills (`.github/skills/{collection-id}/`) - Self-contained skill packages, by convention organized by collection. +* Hooks (`.github/hooks/{collection-id}/`) - Collection-scoped Copilot hook manifests (JSON) that wire lifecycle event commands. * Extension (`extension/`) - VS Code extension source and packaging. * GitHub Configuration (`.github/`) - Workflows, instructions, prompts, agents, composite actions, and issue templates, typically organized into `{collection-id}` subdirectories. * Collections (`collections/`) - YAML and markdown manifests defining bundled sets of agents, prompts, instructions, and skills. @@ -152,6 +153,7 @@ Commit message scopes map to repository directories: * `(prompts)` = `.github/prompts/` * `(instructions)` = `.github/instructions/` * `(skills)` = `.github/skills/` +* `(hooks)` = `.github/hooks/` * `(templates)` = `.github/ISSUE_TEMPLATE/` * `(workflows)` = `.github/workflows/` * `(extension)` = `extension/` diff --git a/.github/skills/installer/hve-core-installer/SKILL.md b/.github/skills/installer/hve-core-installer/SKILL.md index 68e0e78d2..a49e9a203 100644 --- a/.github/skills/installer/hve-core-installer/SKILL.md +++ b/.github/skills/installer/hve-core-installer/SKILL.md @@ -197,6 +197,11 @@ The HVE Core extension has been installed from the VS Code Marketplace. • github-backlog-manager, adr-creation, doc-ops, pr-review • prompt-builder, memory, and more! +🪝 Hooks (manual step): The Marketplace extension is declarative and does not + write chat.hookFilesLocations. To enable bundled hooks (e.g. telemetry), add + each collection's hook folder to that setting yourself, or use a clone-based + or CLI-plugin install which documents this configuration. + 📋 Configuring optional settings... ``` @@ -384,7 +389,7 @@ For Bash: Use `set -euo pipefail`, `test -d` for existence checks, and `echo` fo After cloning, update `.vscode/settings.json` with entries for each collection subdirectory. Replace `` with the settings path prefix from the method table. Do not use `**` glob patterns in paths because `chat.*Locations` settings do not support them. -Enumerate each collection subdirectory under `.github/agents/`, `.github/prompts/`, and `.github/instructions/` from the cloned HVE-Core directory. Create one entry per subdirectory. For `.github/agents/`, also check each collection folder for a `subagents/` subfolder and include it when present (e.g., `hve-core/subagents`). For `.github/skills/`, list only the collection-level folders directly under `.github/skills/` (e.g., `shared`); do not enumerate deeper subfolders (individual skill directories like `shared/pr-reference/` are not listed). Exclude the `installer` collection from `chat.agentSkillsLocations` because it is the installer skill itself and not intended for end-user settings. +Enumerate each collection subdirectory under `.github/agents/`, `.github/prompts/`, `.github/instructions/`, and `.github/hooks/` from the cloned HVE-Core directory. Create one entry per subdirectory. For `.github/agents/`, also check each collection folder for a `subagents/` subfolder and include it when present (e.g., `hve-core/subagents`). For `.github/skills/`, list only the collection-level folders directly under `.github/skills/` (e.g., `shared`); do not enumerate deeper subfolders (individual skill directories like `shared/pr-reference/` are not listed). For `.github/hooks/`, list only the collection-level folders directly under `.github/hooks/` (e.g., `shared`); the default `chat.hookFilesLocations` value only covers the workspace `.github/hooks`, so clone-based installs must add each collection's hook folder explicitly. Exclude the `installer` collection from `chat.agentSkillsLocations` because it is the installer skill itself and not intended for end-user settings. Any folder named `experimental` under any artifact type (agents, prompts, instructions, or skills) must not be included without first asking the user whether they want experimental features. If the user opts in, add the `experimental` entries (and `experimental/subagents` for agents when that subfolder exists). @@ -425,6 +430,9 @@ Any folder named `experimental` under any artifact type (agents, prompts, instru "/.github/skills/rai": true, "/.github/skills/security": true, "/.github/skills/shared": true + }, + "chat.hookFilesLocations": { + "/.github/hooks/shared": true } } ``` @@ -517,6 +525,9 @@ Add to devcontainer.json: "/workspaces/hve-core/.github/skills/rai": true, "/workspaces/hve-core/.github/skills/security": true, "/workspaces/hve-core/.github/skills/shared": true + }, + "chat.hookFilesLocations": { + "/workspaces/hve-core/.github/hooks/shared": true } } } diff --git a/.github/workflows/plugin-validation.yml b/.github/workflows/plugin-validation.yml index 4d4fcf36d..65465f355 100644 --- a/.github/workflows/plugin-validation.yml +++ b/.github/workflows/plugin-validation.yml @@ -45,6 +45,9 @@ jobs: - name: Validate marketplace manifest run: npm run lint:marketplace + - name: Validate hook manifests + run: npm run lint:hooks + - name: Check plugin freshness run: | npm run plugin:generate diff --git a/docs/contributing/hooks.md b/docs/contributing/hooks.md index 25740c5ac..74cb709aa 100644 --- a/docs/contributing/hooks.md +++ b/docs/contributing/hooks.md @@ -3,7 +3,7 @@ title: Contributing Hooks description: How to implement, register, and validate hook artifacts in hve-core sidebar_position: 7 author: Microsoft -ms.date: 2026-06-17 +ms.date: 2026-06-18 ms.topic: how-to keywords: - hooks @@ -27,12 +27,13 @@ Use a hook when you need event-driven behavior such as: Hooks are collection-scoped, like every other distributable artifact type. Use this structure for hook contributions: -| Path | Purpose | -|---|---| -| `.github/hooks//.json` | Hook manifest that maps lifecycle events to executable commands | -| `.github/hooks///` | Hook implementation scripts and support files | -| `collections/*.collection.yml` | Collection registration with `kind: hook` | -| `collections/*.collection.md` | Human-readable hook entry in the collection documentation table | +| Path | Purpose | +|-----------------------------------------------------|-----------------------------------------------------------------| +| `.github/hooks//.json` | Hook manifest that maps lifecycle events to executable commands | +| `.github/hooks///` | Hook implementation scripts and support files | +| `scripts/linting/schemas/hook-manifest.schema.json` | JSON Schema (draft-07) that defines the manifest contract | +| `collections/*.collection.yml` | Collection registration with `kind: hook` | +| `collections/*.collection.md` | Human-readable hook entry in the collection documentation table | Manifests live one collection level down (`.github/hooks//`) so the installer can activate each collection's hooks independently by adding only that collection's folder to `chat.hookFilesLocations`. A flat `.github/hooks/.json` is treated as a repo-specific artifact and is excluded from distribution. @@ -80,6 +81,12 @@ For reliability and portability, hook scripts should follow these rules: Telemetry follows this model with a no-op gate and structured JSONL append behavior. +## Manifest Schema and Validation + +Manifests are validated against `scripts/linting/schemas/hook-manifest.schema.json`, the authoritative contract. The schema enforces the allowed top-level keys (`version`, `description`, `hooks`), the eight CLI-lowercase event names (`sessionStart`, `userPromptSubmit`, `preToolUse`, `postToolUse`, `preCompact`, `subagentStart`, `subagentStop`, `stop`), and the permitted command properties. + +Run `npm run lint:hooks` to validate every collection-scoped manifest. On failure, the validator prints each error and the schema path so you can reconcile the manifest against the contract. + ## Handling Sensitive Payloads Hook payloads can contain sensitive data. `PreToolUse` inputs include full file @@ -126,9 +133,10 @@ Then update the corresponding collection markdown (`collections/*.collection.md` Before opening a PR: -1. Run `npm run plugin:validate` -2. Run `npm run plugin:generate` -3. Run `npm run lint:md` +1. Run `npm run lint:hooks` +2. Run `npm run plugin:validate` +3. Run `npm run plugin:generate` +4. Run `npm run lint:md` When your hook includes scripts, also run the relevant script linters and tests for those languages. diff --git a/package.json b/package.json index db7baa08f..4730839f8 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "lint:adr-consistency": "pwsh -NoProfile -File scripts/linting/Validate-AdrConsistency.ps1 -Paths docs/planning/adrs", "lint:collections-metadata": "pwsh -NoProfile -Command \"./scripts/collections/Validate-Collections.ps1 -OutputPath logs/collection-validation-results.json\"", "lint:marketplace": "pwsh -File scripts/plugins/Validate-Marketplace.ps1 -OutputPath logs/marketplace-validation-results.json", + "lint:hooks": "pwsh -File scripts/linting/Validate-HookManifests.ps1 -OutputPath logs/hook-manifest-validation-results.json", "lint:version-consistency": "pwsh -NoProfile -Command \"./scripts/security/Test-ActionVersionConsistency.ps1 -FailOnMismatch -Format Json -OutputPath logs/action-version-consistency-results.json\"", "lint:permissions": "pwsh -NoProfile -Command \"& './scripts/security/Test-WorkflowPermissions.ps1' -FailOnViolation\"", "lint:dependency-pinning": "pwsh -NoProfile -Command \"& './scripts/security/Test-DependencyPinning.ps1' -FailOnUnpinned\"", @@ -33,7 +34,7 @@ "lint:ai-artifacts": "pwsh -NoProfile -Command \"& './scripts/linting/Validate-PlannerArtifacts.ps1' -FailOnMissing\"", "lint:models": "pwsh -NoProfile -File scripts/linting/Test-ModelReferences.ps1 -OutputPath logs/model-validation-results.json", "lint:models:refresh": "pwsh -NoProfile -File scripts/linting/Update-ModelCatalog.ps1", - "lint:all": "npm run format:tables && npm run lint:md && npm run lint:ps && npm run lint:yaml && npm run lint:json && npm run lint:links && npm run lint:frontmatter && npm run lint:adr-consistency && npm run lint:collections-metadata && npm run lint:marketplace && npm run lint:version-consistency && npm run lint:permissions && npm run lint:dependency-pinning && npm run lint:ps-module-pins && npm run lint:py && npm run validate:skills && npm run lint:ai-artifacts && npm run lint:models && npm run validate:devcontainer-lockfile && npm run eval:lint:vally && npm run eval:lint:schema && npm run eval:lint:text && npm run eval:lint:safety", + "lint:all": "npm run format:tables && npm run lint:md && npm run lint:ps && npm run lint:yaml && npm run lint:json && npm run lint:links && npm run lint:frontmatter && npm run lint:adr-consistency && npm run lint:collections-metadata && npm run lint:marketplace && npm run lint:hooks && npm run lint:version-consistency && npm run lint:permissions && npm run lint:dependency-pinning && npm run lint:ps-module-pins && npm run lint:py && npm run validate:skills && npm run lint:ai-artifacts && npm run lint:models && npm run validate:devcontainer-lockfile && npm run eval:lint:vally && npm run eval:lint:schema && npm run eval:lint:text && npm run eval:lint:safety", "format:tables": "pwsh -NoProfile -File scripts/linting/Format-MarkdownTables.ps1", "extension:prepare": "pwsh ./scripts/extension/Prepare-Extension.ps1 && npm run extension:postprocess", "extension:prepare:prerelease": "pwsh ./scripts/extension/Prepare-Extension.ps1 -Channel PreRelease && npm run extension:postprocess", diff --git a/scripts/linting/Validate-HookManifests.ps1 b/scripts/linting/Validate-HookManifests.ps1 new file mode 100644 index 000000000..e9dbf20e1 --- /dev/null +++ b/scripts/linting/Validate-HookManifests.ps1 @@ -0,0 +1,363 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +#Requires -Version 7.0 + +<# +.SYNOPSIS + Validates collection-scoped hook manifests under .github/hooks/. + +.DESCRIPTION + Discovers hook manifests at .github/hooks//.json and + validates them against the hook manifest contract: required fields, + permitted top-level keys, lifecycle event names (Copilot CLI lowercase + form only), and per-command properties. Declaring an event in both the + CLI-lowercase and PascalCase form is rejected so each event fires once. + +.EXAMPLE + ./Validate-HookManifests.ps1 -OutputPath 'logs/hook-manifest-validation-results.json' +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'logs/hook-manifest-validation-results.json' +) + +$ErrorActionPreference = 'Stop' + +Import-Module (Join-Path $PSScriptRoot '../lib/Modules/CIHelpers.psm1') -Force + +#region Contract + +$script:HookAllowedEvents = @( + 'sessionStart', + 'userPromptSubmit', + 'preToolUse', + 'postToolUse', + 'preCompact', + 'subagentStart', + 'subagentStop', + 'stop' +) + +$script:HookAllowedTopLevel = @('version', 'description', 'hooks') +$script:HookAllowedCommandProps = @('type', 'command', 'bash', 'powershell', 'windows', 'linux', 'osx', 'cwd', 'env', 'timeout', 'timeoutSec') +$script:HookCommandProps = @('command', 'bash', 'powershell', 'windows', 'linux', 'osx') +$script:HookSchemaRelativePath = 'scripts/linting/schemas/hook-manifest.schema.json' + +#endregion Contract + +#region Validation Helpers + +function Test-HookManifest { + <# + .SYNOPSIS + Validates a parsed hook manifest against the hook manifest contract. + + .PARAMETER Manifest + Parsed manifest as a hashtable (from ConvertFrom-Json -AsHashtable). + + .OUTPUTS + [string[]] Array of validation error messages. Empty when valid. + + .EXAMPLE + Test-HookManifest -Manifest $manifest + #> + [CmdletBinding()] + [OutputType([string[]])] + param( + [Parameter(Mandatory = $true)] + $Manifest + ) + + $errors = [System.Collections.Generic.List[string]]::new() + + if ($Manifest -isnot [System.Collections.IDictionary]) { + $errors.Add('manifest must be a JSON object') + return $errors.ToArray() + } + + # Unknown top-level keys + foreach ($key in $Manifest.Keys) { + if ($script:HookAllowedTopLevel -notcontains $key) { + $errors.Add("unknown top-level field '$key'") + } + } + + # version + if (-not $Manifest.ContainsKey('version') -or $null -eq $Manifest['version']) { + $errors.Add("missing required field 'version'") + } + elseif ([int]$Manifest['version'] -ne 1) { + $errors.Add("field 'version' must be 1") + } + + # description (optional) + if ($Manifest.ContainsKey('description') -and [string]::IsNullOrWhiteSpace([string]$Manifest['description'])) { + $errors.Add("field 'description' must be a non-empty string when present") + } + + # hooks + if (-not $Manifest.ContainsKey('hooks') -or $null -eq $Manifest['hooks']) { + $errors.Add("missing required field 'hooks'") + return $errors.ToArray() + } + + $hooks = $Manifest['hooks'] + if ($hooks -isnot [System.Collections.IDictionary]) { + $errors.Add("field 'hooks' must be an object") + return $errors.ToArray() + } + + if ($hooks.Keys.Count -eq 0) { + $errors.Add("field 'hooks' must declare at least one event") + } + + $lowerToCanonical = @{} + foreach ($canonicalEvent in $script:HookAllowedEvents) { + $lowerToCanonical[$canonicalEvent.ToLowerInvariant()] = $canonicalEvent + } + + foreach ($eventName in $hooks.Keys) { + if ($script:HookAllowedEvents -ccontains $eventName) { + # canonical CLI-lowercase form + } + elseif ($lowerToCanonical.ContainsKey($eventName.ToLowerInvariant())) { + $errors.Add("event '$eventName' must use the Copilot CLI lowercase form '$($lowerToCanonical[$eventName.ToLowerInvariant()])'") + continue + } + else { + $errors.Add("unknown event '$eventName'") + continue + } + + $entries = $hooks[$eventName] + if ($entries -isnot [System.Collections.IEnumerable] -or $entries -is [string] -or $entries -is [System.Collections.IDictionary]) { + $errors.Add("event '$eventName' must be an array of command entries") + continue + } + + $entryList = @($entries) + if ($entryList.Count -eq 0) { + $errors.Add("event '$eventName' must declare at least one command entry") + continue + } + + $index = 0 + foreach ($entry in $entryList) { + $label = "event '$eventName' entry [$index]" + $index++ + + if ($entry -isnot [System.Collections.IDictionary]) { + $errors.Add("$label must be an object") + continue + } + + foreach ($prop in $entry.Keys) { + if ($script:HookAllowedCommandProps -notcontains $prop) { + $errors.Add("$label has unknown property '$prop'") + } + } + + if (-not $entry.ContainsKey('type')) { + $errors.Add("$label missing required property 'type'") + } + elseif ([string]$entry['type'] -ne 'command') { + $errors.Add("$label property 'type' must be 'command'") + } + + $hasCommand = $false + foreach ($commandProp in $script:HookCommandProps) { + if ($entry.ContainsKey($commandProp)) { + if ([string]::IsNullOrWhiteSpace([string]$entry[$commandProp])) { + $errors.Add("$label property '$commandProp' must be a non-empty string") + } + else { + $hasCommand = $true + } + } + } + + if (-not $hasCommand) { + $errors.Add("$label must define at least one command property ($($script:HookCommandProps -join ', '))") + } + } + } + + return $errors.ToArray() +} + +function Write-HookValidationReport { + <# + .SYNOPSIS + Writes hook manifest validation results to a JSON report. + + .PARAMETER RepoRoot + Absolute path to the repository root directory. + + .PARAMETER OutputPath + Output report path, absolute or relative to RepoRoot. + + .PARAMETER ErrorCount + Total number of validation errors. + + .PARAMETER Results + Validation results grouped by manifest. + + .OUTPUTS + [void] + #> + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$RepoRoot, + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'logs/hook-manifest-validation-results.json', + + [Parameter(Mandatory = $true)] + [int]$ErrorCount, + + [Parameter(Mandatory = $false)] + [array]$Results = @() + ) + + if ([string]::IsNullOrWhiteSpace($OutputPath)) { + return + } + + $resolvedOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) { + $OutputPath + } + else { + Join-Path -Path $RepoRoot -ChildPath $OutputPath + } + + $outputDirectory = Split-Path -Path $resolvedOutputPath -Parent + if (-not [string]::IsNullOrWhiteSpace($outputDirectory) -and -not (Test-Path -Path $outputDirectory -PathType Container)) { + New-Item -Path $outputDirectory -ItemType Directory -Force | Out-Null + } + + $report = [ordered]@{ + Timestamp = (Get-Date).ToUniversalTime().ToString('o') + Schema = $script:HookSchemaRelativePath + ErrorCount = $ErrorCount + Results = @($Results) + } + + $report | ConvertTo-Json -Depth 10 | Set-Content -Path $resolvedOutputPath -Encoding UTF8 +} + +#endregion Validation Helpers + +#region Orchestration + +function Invoke-HookManifestValidation { + <# + .SYNOPSIS + Validates all collection-scoped hook manifests in the repository. + + .PARAMETER RepoRoot + Absolute path to the repository root directory. + + .PARAMETER OutputPath + Output report path, absolute or relative to RepoRoot. + + .OUTPUTS + Hashtable with Success bool and ErrorCount int. + #> + [CmdletBinding()] + [OutputType([hashtable])] + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$RepoRoot, + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'logs/hook-manifest-validation-results.json' + ) + + $hooksRoot = Join-Path -Path $RepoRoot -ChildPath '.github' -AdditionalChildPath 'hooks' + + if (-not (Test-Path -Path $hooksRoot -PathType Container)) { + Write-Host 'No .github/hooks directory found; nothing to validate.' + Write-HookValidationReport -RepoRoot $RepoRoot -OutputPath $OutputPath -ErrorCount 0 -Results @() + return @{ Success = $true; ErrorCount = 0 } + } + + # Collection-scoped manifests live at .github/hooks//.json. + $manifestFiles = @(Get-ChildItem -Path $hooksRoot -Filter '*.json' -File -Recurse | + Where-Object { $_.Directory.Parent.FullName -eq (Get-Item $hooksRoot).FullName }) + + Write-Host "Validating hook manifests ($($manifestFiles.Count) found)..." + + $totalErrors = 0 + $results = @() + + foreach ($file in $manifestFiles) { + $relativePath = [System.IO.Path]::GetRelativePath($RepoRoot, $file.FullName) + $fileErrors = @() + + try { + $manifest = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json -AsHashtable + } + catch { + $fileErrors = @("invalid JSON: $($_.Exception.Message)") + } + + if ($fileErrors.Count -eq 0) { + $fileErrors = @(Test-HookManifest -Manifest $manifest) + } + + $results += @{ + Manifest = $relativePath + IsValid = ($fileErrors.Count -eq 0) + Errors = @($fileErrors) + } + + if ($fileErrors.Count -gt 0) { + $totalErrors += $fileErrors.Count + Write-Host " FAIL $relativePath - $($fileErrors.Count) error(s)" -ForegroundColor Red + foreach ($err in $fileErrors) { + Write-Host " $err" -ForegroundColor Red + } + Write-Host " See contract: $script:HookSchemaRelativePath" -ForegroundColor Yellow + } + else { + Write-Host " OK $relativePath" + } + } + + Write-HookValidationReport -RepoRoot $RepoRoot -OutputPath $OutputPath -ErrorCount $totalErrors -Results $results + + return @{ + Success = ($totalErrors -eq 0) + ErrorCount = $totalErrors + } +} + +#endregion Orchestration + +#region Main Execution +if ($MyInvocation.InvocationName -ne '.') { + try { + $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path + $RepoRoot = (Get-Item "$ScriptDir/../..").FullName + + $result = Invoke-HookManifestValidation -RepoRoot $RepoRoot -OutputPath $OutputPath + + if (-not $result.Success) { + throw "Hook manifest validation failed with $($result.ErrorCount) error(s)." + } + + exit 0 + } + catch { + Write-Error "Hook manifest validation failed: $($_.Exception.Message)" + Write-CIAnnotation -Message $_.Exception.Message -Level Error + exit 1 + } +} +#endregion diff --git a/scripts/linting/schemas/hook-manifest.schema.json b/scripts/linting/schemas/hook-manifest.schema.json new file mode 100644 index 000000000..e278f7164 --- /dev/null +++ b/scripts/linting/schemas/hook-manifest.schema.json @@ -0,0 +1,144 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hook Manifest", + "description": "Schema for hve-core collection-scoped hook manifests under .github/hooks//.json. Event keys use the Copilot CLI lowercase form; VS Code converts them to PascalCase at load time. Declaring an event in both CLI-lowercase and PascalCase form is rejected so each event fires once.", + "type": "object", + "required": [ + "version", + "hooks" + ], + "properties": { + "version": { + "type": "integer", + "enum": [ + 1 + ], + "description": "Manifest schema version. Only version 1 is currently defined." + }, + "description": { + "type": "string", + "minLength": 1, + "description": "Short description of what the hook does." + }, + "hooks": { + "type": "object", + "minProperties": 1, + "description": "Map of lifecycle event names to arrays of hook command entries.", + "propertyNames": { + "enum": [ + "sessionStart", + "userPromptSubmit", + "preToolUse", + "postToolUse", + "preCompact", + "subagentStart", + "subagentStop", + "stop" + ] + }, + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/hookCommand" + } + } + } + }, + "additionalProperties": false, + "definitions": { + "hookCommand": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "command", + "description": "Hook entry type. Only \"command\" is supported." + }, + "command": { + "type": "string", + "minLength": 1, + "description": "Cross-platform command. VS Code-native property." + }, + "bash": { + "type": "string", + "minLength": 1, + "description": "Copilot CLI property mapped to the osx and linux commands." + }, + "powershell": { + "type": "string", + "minLength": 1, + "description": "Copilot CLI property mapped to the windows command." + }, + "windows": { + "type": "string", + "minLength": 1, + "description": "Windows-specific command override. VS Code-native property." + }, + "linux": { + "type": "string", + "minLength": 1, + "description": "Linux-specific command override. VS Code-native property." + }, + "osx": { + "type": "string", + "minLength": 1, + "description": "macOS-specific command override. VS Code-native property." + }, + "cwd": { + "type": "string", + "description": "Working directory relative to the repository root." + }, + "env": { + "type": "object", + "description": "Additional environment variables for the command." + }, + "timeout": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Timeout in seconds. VS Code-native property (default 30)." + }, + "timeoutSec": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Timeout in seconds. Copilot CLI property." + } + }, + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "command" + ] + }, + { + "required": [ + "bash" + ] + }, + { + "required": [ + "powershell" + ] + }, + { + "required": [ + "windows" + ] + }, + { + "required": [ + "linux" + ] + }, + { + "required": [ + "osx" + ] + } + ] + } + } +} diff --git a/scripts/tests/linting/Validate-HookManifests.Tests.ps1 b/scripts/tests/linting/Validate-HookManifests.Tests.ps1 new file mode 100644 index 000000000..629810e65 --- /dev/null +++ b/scripts/tests/linting/Validate-HookManifests.Tests.ps1 @@ -0,0 +1,189 @@ +#Requires -Modules Pester +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +BeforeAll { + . $PSScriptRoot/../../linting/Validate-HookManifests.ps1 +} + +Describe 'Test-HookManifest - valid manifests' { + It 'Returns no errors for a valid CLI-form manifest' { + $manifest = @{ + version = 1 + description = 'Telemetry collector' + hooks = @{ + sessionStart = @( + @{ type = 'command'; bash = './collect.sh'; powershell = './Collect.ps1'; timeoutSec = 10 } + ) + } + } + + Test-HookManifest -Manifest $manifest | Should -BeNullOrEmpty + } + + It 'Returns no errors for a valid VS Code-native command manifest' { + $manifest = @{ + version = 1 + hooks = @{ + stop = @( + @{ type = 'command'; command = './done.sh'; timeout = 5 } + ) + } + } + + Test-HookManifest -Manifest $manifest | Should -BeNullOrEmpty + } + + It 'Accepts all eight lifecycle events' { + $manifest = @{ + version = 1 + hooks = @{ + sessionStart = @(@{ type = 'command'; bash = 'a' }) + userPromptSubmit = @(@{ type = 'command'; bash = 'b' }) + preToolUse = @(@{ type = 'command'; bash = 'c' }) + postToolUse = @(@{ type = 'command'; bash = 'd' }) + preCompact = @(@{ type = 'command'; bash = 'e' }) + subagentStart = @(@{ type = 'command'; bash = 'f' }) + subagentStop = @(@{ type = 'command'; bash = 'g' }) + stop = @(@{ type = 'command'; bash = 'h' }) + } + } + + Test-HookManifest -Manifest $manifest | Should -BeNullOrEmpty + } +} + +Describe 'Test-HookManifest - structural errors' { + It 'Reports missing version' { + $manifest = @{ hooks = @{ stop = @(@{ type = 'command'; bash = 'a' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "missing required field 'version'" + } + + It 'Reports unsupported version' { + $manifest = @{ version = 2; hooks = @{ stop = @(@{ type = 'command'; bash = 'a' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "field 'version' must be 1" + } + + It 'Reports missing hooks' { + $manifest = @{ version = 1 } + Test-HookManifest -Manifest $manifest | Should -Contain "missing required field 'hooks'" + } + + It 'Reports unknown top-level field' { + $manifest = @{ version = 1; extra = 'nope'; hooks = @{ stop = @(@{ type = 'command'; bash = 'a' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "unknown top-level field 'extra'" + } + + It 'Reports empty hooks object' { + $manifest = @{ version = 1; hooks = @{} } + Test-HookManifest -Manifest $manifest | Should -Contain "field 'hooks' must declare at least one event" + } +} + +Describe 'Test-HookManifest - event name enforcement' { + It 'Rejects the PascalCase form and points to the CLI lowercase form' { + $manifest = @{ version = 1; hooks = @{ SessionStart = @(@{ type = 'command'; bash = 'a' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "event 'SessionStart' must use the Copilot CLI lowercase form 'sessionStart'" + } + + It 'Rejects an unknown event name' { + $manifest = @{ version = 1; hooks = @{ onSomething = @(@{ type = 'command'; bash = 'a' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "unknown event 'onSomething'" + } +} + +Describe 'Test-HookManifest - command entry errors' { + It 'Reports a non-array event value' { + $manifest = @{ version = 1; hooks = @{ stop = @{ type = 'command'; bash = 'a' } } } + Test-HookManifest -Manifest $manifest | Should -Contain "event 'stop' must be an array of command entries" + } + + It 'Reports an empty event array' { + $manifest = @{ version = 1; hooks = @{ stop = @() } } + Test-HookManifest -Manifest $manifest | Should -Contain "event 'stop' must declare at least one command entry" + } + + It 'Reports a missing type property' { + $manifest = @{ version = 1; hooks = @{ stop = @(@{ bash = 'a' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "event 'stop' entry [0] missing required property 'type'" + } + + It 'Reports a non-command type' { + $manifest = @{ version = 1; hooks = @{ stop = @(@{ type = 'script'; bash = 'a' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "event 'stop' entry [0] property 'type' must be 'command'" + } + + It 'Reports an unknown command property' { + $manifest = @{ version = 1; hooks = @{ stop = @(@{ type = 'command'; bash = 'a'; nope = 'x' }) } } + Test-HookManifest -Manifest $manifest | Should -Contain "event 'stop' entry [0] has unknown property 'nope'" + } + + It 'Reports an entry with no command property' { + $manifest = @{ version = 1; hooks = @{ stop = @(@{ type = 'command' }) } } + $errors = Test-HookManifest -Manifest $manifest + ($errors -join "`n") | Should -BeLike '*must define at least one command property*' + } +} + +Describe 'Invoke-HookManifestValidation' { + It 'Succeeds when no hooks directory exists' { + $repoRoot = Join-Path $TestDrive 'repo-no-hooks' + New-Item -ItemType Directory -Path $repoRoot -Force | Out-Null + + $result = Invoke-HookManifestValidation -RepoRoot $repoRoot -OutputPath ([System.IO.Path]::Combine($repoRoot, 'logs', 'out.json')) + $result.Success | Should -BeTrue + $result.ErrorCount | Should -Be 0 + } + + It 'Writes the schema contract path into the report' { + $repoRoot = Join-Path $TestDrive 'repo-schema-ref' + New-Item -ItemType Directory -Path $repoRoot -Force | Out-Null + $outPath = [System.IO.Path]::Combine($repoRoot, 'logs', 'out.json') + + Invoke-HookManifestValidation -RepoRoot $repoRoot -OutputPath $outPath | Out-Null + + $report = Get-Content -Path $outPath -Raw | ConvertFrom-Json + $report.Schema | Should -Be 'scripts/linting/schemas/hook-manifest.schema.json' + } + + It 'Validates a collection-scoped manifest and ignores deeper files' { + $repoRoot = Join-Path $TestDrive 'repo-hooks' + $collectionDir = Join-Path $repoRoot '.github/hooks/shared' + $implDir = Join-Path $collectionDir 'telemetry' + New-Item -ItemType Directory -Path $implDir -Force | Out-Null + + $manifest = @{ + version = 1 + hooks = @{ sessionStart = @(@{ type = 'command'; bash = './a.sh' }) } + } + Set-Content -Path (Join-Path $collectionDir 'telemetry.json') -Value ($manifest | ConvertTo-Json -Depth 10) + # A deeper JSON file (implementation config) must not be treated as a manifest. + Set-Content -Path (Join-Path $implDir 'config.json') -Value '{ "not": "a manifest" }' + + $result = Invoke-HookManifestValidation -RepoRoot $repoRoot -OutputPath ([System.IO.Path]::Combine($repoRoot, 'logs', 'out.json')) + $result.Success | Should -BeTrue + $result.ErrorCount | Should -Be 0 + } + + It 'Fails on an invalid collection-scoped manifest' { + $repoRoot = Join-Path $TestDrive 'repo-bad-hooks' + $collectionDir = Join-Path $repoRoot '.github/hooks/shared' + New-Item -ItemType Directory -Path $collectionDir -Force | Out-Null + Set-Content -Path (Join-Path $collectionDir 'telemetry.json') -Value '{ "hooks": { "SessionStart": [ { "type": "command", "bash": "a" } ] } }' + + $result = Invoke-HookManifestValidation -RepoRoot $repoRoot -OutputPath ([System.IO.Path]::Combine($repoRoot, 'logs', 'out.json')) + $result.Success | Should -BeFalse + $result.ErrorCount | Should -BeGreaterThan 0 + } + + It 'Rejects a manifest that declares an event in both CLI-lowercase and PascalCase form' { + $repoRoot = Join-Path $TestDrive 'repo-both-forms' + $collectionDir = Join-Path $repoRoot '.github/hooks/shared' + New-Item -ItemType Directory -Path $collectionDir -Force | Out-Null + Set-Content -Path (Join-Path $collectionDir 'telemetry.json') -Value '{ "version": 1, "hooks": { "stop": [ { "type": "command", "bash": "a" } ], "Stop": [ { "type": "command", "bash": "b" } ] } }' + + $result = Invoke-HookManifestValidation -RepoRoot $repoRoot -OutputPath ([System.IO.Path]::Combine($repoRoot, 'logs', 'out.json')) + $result.Success | Should -BeFalse + $result.ErrorCount | Should -BeGreaterThan 0 + } +} diff --git a/scripts/tests/plugins/PluginHelpers.Tests.ps1 b/scripts/tests/plugins/PluginHelpers.Tests.ps1 index 686e4f837..ef54790cd 100644 --- a/scripts/tests/plugins/PluginHelpers.Tests.ps1 +++ b/scripts/tests/plugins/PluginHelpers.Tests.ps1 @@ -522,3 +522,54 @@ Describe 'Repair-PluginSymlinkIndex' { } } } + +Describe 'Get-PluginItemName - hook kind' { + It 'Returns the filename unchanged for a hook' { + Get-PluginItemName -FileName 'telemetry.json' -Kind 'hook' | Should -Be 'telemetry.json' + } +} + +Describe 'Get-PluginItemSubpath - hook kind' { + It 'Strips the .github/hooks prefix and returns the collection subpath' { + $result = Get-PluginItemSubpath -Path '.github/hooks/shared/telemetry.json' -Kind 'hook' + $result | Should -Be 'shared' + } + + It 'Returns the nested subpath for deeper hook layouts' { + $result = Get-PluginItemSubpath -Path '.github/hooks/shared/telemetry/config.json' -Kind 'hook' + $result | Should -Be 'shared/telemetry' + } + + It 'Returns empty string for a hook directly under the kind root' { + $result = Get-PluginItemSubpath -Path '.github/hooks/telemetry.json' -Kind 'hook' + $result | Should -Be '' + } +} + +Describe 'Get-PluginSubdirectory - hook kind' { + It 'Returns hooks for the hook kind' { + Get-PluginSubdirectory -Kind 'hook' | Should -Be 'hooks' + } +} + +Describe 'New-PluginManifestContent - hook paths' { + It 'Emits a single hooks string for one hook path' { + $manifest = New-PluginManifestContent -CollectionId 'shared' -Description 'desc' -Version '1.0.0' -HookPaths @('hooks/shared/telemetry.json') + $manifest['hooks'] | Should -BeOfType [string] + $manifest['hooks'] | Should -Be 'hooks/shared/telemetry.json' + } + + It 'Uses the first sorted hook path and warns when multiple are declared' { + $warnings = $null + $manifest = New-PluginManifestContent -CollectionId 'shared' -Description 'desc' -Version '1.0.0' ` + -HookPaths @('hooks/shared/zeta.json', 'hooks/shared/alpha.json') -WarningVariable warnings -WarningAction SilentlyContinue + $manifest['hooks'] | Should -Be 'hooks/shared/alpha.json' + $warnings | Should -Not -BeNullOrEmpty + ($warnings -join "`n") | Should -Match 'references only one' + } + + It 'Omits the hooks key when no hook paths are provided' { + $manifest = New-PluginManifestContent -CollectionId 'shared' -Description 'desc' -Version '1.0.0' + $manifest.Contains('hooks') | Should -BeFalse + } +} From ed6371d842c95e4e4f0d35d657a8da91fdae4476 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 18 Jun 2026 16:38:22 -0600 Subject: [PATCH 05/10] format fixes --- .../hooks/shared/telemetry/_telemetry_core.py | 52 ++++++------------- .../shared/telemetry/tests/fuzz_harness.py | 11 +++- .../telemetry/tests/test_telemetry_core.py | 22 ++------ docs/customization/README.md | 24 ++++----- 4 files changed, 43 insertions(+), 66 deletions(-) diff --git a/.github/hooks/shared/telemetry/_telemetry_core.py b/.github/hooks/shared/telemetry/_telemetry_core.py index b34cc34d0..561cf4edf 100644 --- a/.github/hooks/shared/telemetry/_telemetry_core.py +++ b/.github/hooks/shared/telemetry/_telemetry_core.py @@ -76,9 +76,7 @@ def _is_safe_sid(sid: str) -> bool: Rejects empty ids and any value containing a path separator or ``..`` traversal sequence so callers never build paths outside their store. """ - return bool(sid) and not ( - os.sep in sid or "/" in sid or "\\" in sid or ".." in sid - ) + return bool(sid) and not (os.sep in sid or "/" in sid or "\\" in sid or ".." in sid) def collect_sids(hook_files: Iterable[str]) -> set[str]: @@ -255,9 +253,7 @@ def write_report_launchers(script_dir: Path | None = None) -> None: "__CLEAN_PS1__", clean_ps1.replace("'", "''") ) (home / "generate-report.ps1").write_text(pwsh_text, encoding="utf-8") - (home / "clean-telemetry.ps1").write_text( - pwsh_clean_text, encoding="utf-8" - ) + (home / "clean-telemetry.ps1").write_text(pwsh_clean_text, encoding="utf-8") else: # Shell-quote so unusual install paths (spaces, quotes, ``$``) # cannot break or inject into the generated launchers. @@ -293,9 +289,7 @@ def find_process_log(state_dir: Path, home: Path) -> str | None: return candidates[0] if candidates else None -def _log_references_interactions( - log_path: str, interaction_ids: set[str] -) -> bool: +def _log_references_interactions(log_path: str, interaction_ids: set[str]) -> bool: """Return True when a process log references any of the interaction ids. Uses a cheap line substring scan so logs that cannot belong to the session @@ -304,9 +298,7 @@ def _log_references_interactions( try: with open(log_path, encoding="utf-8", errors="replace") as handle: for line in handle: - if "interaction_id" in line and any( - iid in line for iid in interaction_ids - ): + if "interaction_id" in line and any(iid in line for iid in interaction_ids): return True except OSError: return False @@ -547,9 +539,7 @@ def _totals_from_state_fallback(state_file: str | os.PathLike[str]) -> dict: msg_output_total += output_tokens model = data.get("model", "") if model: - msg_output_by_model[model] = ( - msg_output_by_model.get(model, 0) + output_tokens - ) + msg_output_by_model[model] = msg_output_by_model.get(model, 0) + output_tokens continue if etype != "session.shutdown": continue @@ -639,9 +629,7 @@ def build_session_summary( token_source = "process_log" # Compute per-subagent token attribution when subagents were used. if meta["subagent_map"]: - agent_usage = _per_agent_usage_from_process_log( - entries, meta["subagent_map"] - ) + agent_usage = _per_agent_usage_from_process_log(entries, meta["subagent_map"]) if totals is None: totals = _totals_from_state_fallback(state_file) @@ -694,9 +682,7 @@ def _normalize_event(data: dict) -> str: def _normalize_timestamp(raw_ts: object) -> str: """Coerce a hook timestamp (epoch ms or string) to an ISO-8601 string.""" if isinstance(raw_ts, (int, float)): - return datetime.datetime.fromtimestamp( - raw_ts / 1000, tz=datetime.UTC - ).isoformat() + return datetime.datetime.fromtimestamp(raw_ts / 1000, tz=datetime.UTC).isoformat() if isinstance(raw_ts, str) and raw_ts: return raw_ts return datetime.datetime.now(datetime.UTC).isoformat() @@ -723,9 +709,7 @@ class _AgentStack: def __init__(self, stack_dir: Path, sid: str) -> None: self.stack_dir = stack_dir - self.stack_file = ( - stack_dir / f"{sid}.json" if _is_safe_sid(sid) else None - ) + self.stack_file = stack_dir / f"{sid}.json" if _is_safe_sid(sid) else None def _read(self) -> list[str]: if self.stack_file and self.stack_file.exists(): @@ -791,9 +775,7 @@ def build_entry(data: dict, event: str, stack: _AgentStack) -> dict | None: entry["prompt"] = (data.get("prompt", "") or "")[:200] elif event == "PreToolUse": entry["tool"] = tool_name - entry["tool_input_keys"] = ( - list(tool_input.keys()) if isinstance(tool_input, dict) else [] - ) + entry["tool_input_keys"] = list(tool_input.keys()) if isinstance(tool_input, dict) else [] entry["agent"] = stack.current() # Detect instructions and skills by file path convention to track # which artifacts the agent loaded during the session. @@ -819,9 +801,7 @@ def build_entry(data: dict, event: str, stack: _AgentStack) -> dict | None: elif event == "PostToolUse": entry["tool"] = tool_name if isinstance(tool_result, dict): - text = tool_result.get("text_result_for_llm") or tool_result.get( - "textResultForLlm", "" - ) + text = tool_result.get("text_result_for_llm") or tool_result.get("textResultForLlm", "") entry["tool_response_len"] = len(text if isinstance(text, str) else str(text)) elif isinstance(tool_result, str): entry["tool_response_len"] = len(tool_result) @@ -897,8 +877,12 @@ def _mode_collect() -> int: state_file = state_dir / "events.jsonl" if state_file.is_file(): summary = build_session_summary( - sid, state_dir, state_file, home, - ts_override=entry["ts"], client=_detect_client(), + sid, + state_dir, + state_file, + home, + ts_override=entry["ts"], + client=_detect_client(), ) if summary is not None: with open(log_file, "a", encoding="utf-8") as handle: @@ -1013,9 +997,7 @@ def _mode_clean(all_dirs: bool, dry_run: bool) -> int: targets: list[Path] = [] if all_dirs: targets.extend(Path(d) for d in read_registry_dirs()) - targets.append( - Path(os.environ.get("HVE_TELEMETRY_DIR", ".copilot-tracking/telemetry")) - ) + targets.append(Path(os.environ.get("HVE_TELEMETRY_DIR", ".copilot-tracking/telemetry"))) seen: set[str] = set() for tel_dir in targets: diff --git a/.github/hooks/shared/telemetry/tests/fuzz_harness.py b/.github/hooks/shared/telemetry/tests/fuzz_harness.py index 66299ef09..77cc69115 100644 --- a/.github/hooks/shared/telemetry/tests/fuzz_harness.py +++ b/.github/hooks/shared/telemetry/tests/fuzz_harness.py @@ -50,8 +50,15 @@ def fuzz_build_entry(data: bytes) -> None: """Fuzz entry building with arbitrary payloads.""" provider = atheris.FuzzedDataProvider(data) events = [ - "SessionStart", "UserPromptSubmit", "PreToolUse", "PostToolUse", - "SubagentStart", "SubagentStop", "PreCompact", "Stop", "unknown", + "SessionStart", + "UserPromptSubmit", + "PreToolUse", + "PostToolUse", + "SubagentStart", + "SubagentStop", + "PreCompact", + "Stop", + "unknown", ] event = events[provider.ConsumeIntInRange(0, len(events) - 1)] payload = { diff --git a/.github/hooks/shared/telemetry/tests/test_telemetry_core.py b/.github/hooks/shared/telemetry/tests/test_telemetry_core.py index 9de886bca..05ee35bd8 100644 --- a/.github/hooks/shared/telemetry/tests/test_telemetry_core.py +++ b/.github/hooks/shared/telemetry/tests/test_telemetry_core.py @@ -161,9 +161,7 @@ def test_given_process_log_when_build_session_summary_then_uses_process_log(tmp_ }, } ] - home, state_dir, state_file = _make_session( - tmp_path, "sid1", state_rows, process_rows, pid=777 - ) + home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows, process_rows, pid=777) summary = core.build_session_summary("sid1", state_dir, state_file, home) assert summary["input_tokens"] == 10 assert summary["input_tokens_uncached"] == 7 @@ -380,9 +378,7 @@ def test_given_skill_path_when_build_entry_then_detects_skill(tmp_path): data = { "hook_event_name": "PreToolUse", "tool_name": "read", - "tool_input": { - "filePath": "/repo/.github/skills/coll/my-skill/SKILL.md" - }, + "tool_input": {"filePath": "/repo/.github/skills/coll/my-skill/SKILL.md"}, } entry = core.build_entry(data, "PreToolUse", stack) assert entry["skill"] == "my-skill" @@ -576,9 +572,7 @@ def test_given_posix_when_write_report_launchers_then_clean_sh_delegates_to_bash assert not (hve / "clean-telemetry.ps1").exists() -def test_given_windows_when_write_report_launchers_then_clean_ps1_is_native( - tmp_path, monkeypatch -): +def test_given_windows_when_write_report_launchers_then_clean_ps1_is_native(tmp_path, monkeypatch): hve = tmp_path / "hve" script_dir = tmp_path / "hook" script_dir.mkdir() @@ -635,9 +629,7 @@ def test_given_dry_run_when_clean_telemetry_dir_then_reports_without_deleting(tm assert len(removed) == 5 -def test_given_current_store_when_mode_clean_then_cleans_only_current( - tmp_path, monkeypatch -): +def test_given_current_store_when_mode_clean_then_cleans_only_current(tmp_path, monkeypatch): current = tmp_path / "current" other = tmp_path / "other" hve = tmp_path / "hve" @@ -655,9 +647,7 @@ def test_given_current_store_when_mode_clean_then_cleans_only_current( assert registry.exists() -def test_given_all_dirs_when_mode_clean_then_cleans_registry_and_home( - tmp_path, monkeypatch -): +def test_given_all_dirs_when_mode_clean_then_cleans_registry_and_home(tmp_path, monkeypatch): current = tmp_path / "current" other = tmp_path / "other" hve = tmp_path / "hve" @@ -685,5 +675,3 @@ def test_given_clean_mode_when_main_dispatches_then_parses_flags(tmp_path, monke assert core.main(["clean", "--dry-run"]) == 0 # Dry-run leaves artifacts in place. assert (current / "raw-input.jsonl").exists() - - diff --git a/docs/customization/README.md b/docs/customization/README.md index 8082170ed..0579d8f69 100644 --- a/docs/customization/README.md +++ b/docs/customization/README.md @@ -2,7 +2,7 @@ title: Customizing HVE Core description: Overview of customization approaches from lightweight settings to full fork-and-extend, with role-based entry points author: Microsoft -ms.date: 2026-03-10 +ms.date: 2026-06-18 ms.topic: overview keywords: - customization @@ -91,17 +91,17 @@ Each artifact guide below includes an "Accelerating with Prompt Builder" section Each HVE role benefits from different customization techniques. The table below maps the nine roles to the guides most relevant to their workflow. -| Role | Recommended Guides | Rationale | -|--------------------------|----------------------------------------------------------------------------------|---------------------------------------------------------------------------------| -| Engineer | [Instructions](instructions.md), [Agents](custom-agents.md) | Coding standards and specialized review agents accelerate daily development | -| TPM | [Prompts](prompts.md), [Collections](collections.md) | Reusable planning prompts and curated bundles standardize project workflows | -| Tech Lead / Architect | [Instructions](instructions.md), [Agents](custom-agents.md), [Skills](skills.md) | Standards enforcement, architecture review agents, and deep domain knowledge | -| Security Architect | [Skills](skills.md), [Instructions](instructions.md) | Compliance knowledge packages and security-focused coding conventions | -| Data Scientist | [Skills](skills.md), [Prompts](prompts.md) | Analytical domain bundles and repeatable notebook workflows | -| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md), [Local Telemetry](local-telemetry.md) | Infrastructure conventions, DevContainer tuning, and local telemetry workflows | -| Business Program Manager | [Prompts](prompts.md), [Team Adoption](team-adoption.md) | Sprint-planning prompts and governance patterns for stakeholder alignment | -| New Contributor | [Instructions](instructions.md), [Environment](environment.md) | Quick onboarding through conventions and a ready-to-use development environment | -| Utility | [Collections](collections.md), [Build System](build-system.md) | Cross-cutting tooling assembly and validation pipeline customization | +| Role | Recommended Guides | Rationale | +|--------------------------|-------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------| +| Engineer | [Instructions](instructions.md), [Agents](custom-agents.md) | Coding standards and specialized review agents accelerate daily development | +| TPM | [Prompts](prompts.md), [Collections](collections.md) | Reusable planning prompts and curated bundles standardize project workflows | +| Tech Lead / Architect | [Instructions](instructions.md), [Agents](custom-agents.md), [Skills](skills.md) | Standards enforcement, architecture review agents, and deep domain knowledge | +| Security Architect | [Skills](skills.md), [Instructions](instructions.md) | Compliance knowledge packages and security-focused coding conventions | +| Data Scientist | [Skills](skills.md), [Prompts](prompts.md) | Analytical domain bundles and repeatable notebook workflows | +| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md), [Local Telemetry](local-telemetry.md) | Infrastructure conventions, DevContainer tuning, and local telemetry workflows | +| Business Program Manager | [Prompts](prompts.md), [Team Adoption](team-adoption.md) | Sprint-planning prompts and governance patterns for stakeholder alignment | +| New Contributor | [Instructions](instructions.md), [Environment](environment.md) | Quick onboarding through conventions and a ready-to-use development environment | +| Utility | [Collections](collections.md), [Build System](build-system.md) | Cross-cutting tooling assembly and validation pipeline customization | ## File Index From f40c4549c94934973121c0704f58fd5a4fc6a464 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 18 Jun 2026 17:06:02 -0600 Subject: [PATCH 06/10] fix tests for ci --- .github/hooks/shared/telemetry/pyproject.toml | 1 + .../telemetry/tests/corpus/0_jsonl_event | 1 + .../telemetry/tests/corpus/1_event_alias | 1 + .../telemetry/tests/corpus/2_build_entry | 1 + .../shared/telemetry/tests/corpus/README.md | 34 ++++ .../shared/telemetry/tests/fuzz_harness.py | 5 +- .github/hooks/shared/telemetry/uv.lock | 174 ++++++++++++++++++ 7 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 .github/hooks/shared/telemetry/tests/corpus/0_jsonl_event create mode 100644 .github/hooks/shared/telemetry/tests/corpus/1_event_alias create mode 100644 .github/hooks/shared/telemetry/tests/corpus/2_build_entry create mode 100644 .github/hooks/shared/telemetry/tests/corpus/README.md diff --git a/.github/hooks/shared/telemetry/pyproject.toml b/.github/hooks/shared/telemetry/pyproject.toml index 2601a4e3b..b7128ed2c 100644 --- a/.github/hooks/shared/telemetry/pyproject.toml +++ b/.github/hooks/shared/telemetry/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [] [dependency-groups] dev = [ "pytest>=9.0", + "pytest-cov>=7.0", "ruff>=0.15", ] # Atheris ships manylinux-only wheels; keep separate from dev so uv sync works on macOS. diff --git a/.github/hooks/shared/telemetry/tests/corpus/0_jsonl_event b/.github/hooks/shared/telemetry/tests/corpus/0_jsonl_event new file mode 100644 index 000000000..d131b1928 --- /dev/null +++ b/.github/hooks/shared/telemetry/tests/corpus/0_jsonl_event @@ -0,0 +1 @@ +0{"event": "SessionStart", "session_id": "abc", "timestamp": "t"} diff --git a/.github/hooks/shared/telemetry/tests/corpus/1_event_alias b/.github/hooks/shared/telemetry/tests/corpus/1_event_alias new file mode 100644 index 000000000..ee8d65fbd --- /dev/null +++ b/.github/hooks/shared/telemetry/tests/corpus/1_event_alias @@ -0,0 +1 @@ +1sessionStart \ No newline at end of file diff --git a/.github/hooks/shared/telemetry/tests/corpus/2_build_entry b/.github/hooks/shared/telemetry/tests/corpus/2_build_entry new file mode 100644 index 000000000..0eecce576 --- /dev/null +++ b/.github/hooks/shared/telemetry/tests/corpus/2_build_entry @@ -0,0 +1 @@ +2PreToolUse \ No newline at end of file diff --git a/.github/hooks/shared/telemetry/tests/corpus/README.md b/.github/hooks/shared/telemetry/tests/corpus/README.md new file mode 100644 index 000000000..37b581549 --- /dev/null +++ b/.github/hooks/shared/telemetry/tests/corpus/README.md @@ -0,0 +1,34 @@ +--- +title: Fuzz Corpus Seeds +description: Seed inputs for coverage-guided fuzzing with the Atheris fuzz harness +author: Microsoft +ms.date: 2026-06-18 +ms.topic: reference +keywords: + - fuzz + - corpus + - atheris + - telemetry +estimated_reading_time: 2 +--- + + +# Fuzz Corpus Seeds + +Seed inputs for the telemetry Atheris fuzz harness. Each file is raw bytes consumed +by `fuzz_dispatch`, which routes `data[0] % 3` to one of three targets. + +## Naming Convention + +`{target_index}_{description}` where `target_index` matches the `FUZZ_TARGETS` +array position: + +| Index | Target | +|-------|------------------------| +| 0 | `fuzz_iter_jsonl` | +| 1 | `fuzz_normalize_event` | +| 2 | `fuzz_build_entry` | + +The first byte selects the target; the remaining bytes are the input payload. + +*🤖 Crafted with precision by ✨Copilot following brilliant human instruction, then carefully refined by our team of discerning human reviewers.* diff --git a/.github/hooks/shared/telemetry/tests/fuzz_harness.py b/.github/hooks/shared/telemetry/tests/fuzz_harness.py index 77cc69115..26d15270f 100644 --- a/.github/hooks/shared/telemetry/tests/fuzz_harness.py +++ b/.github/hooks/shared/telemetry/tests/fuzz_harness.py @@ -10,8 +10,11 @@ import sys from contextlib import suppress +from pathlib import Path -import _telemetry_core as core +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import _telemetry_core as core # noqa: E402 try: import atheris diff --git a/.github/hooks/shared/telemetry/uv.lock b/.github/hooks/shared/telemetry/uv.lock index deac25054..4a435967a 100644 --- a/.github/hooks/shared/telemetry/uv.lock +++ b/.github/hooks/shared/telemetry/uv.lock @@ -22,6 +22,110 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/d7/477ad149490e6cb849f28abea1dabb9c823cea72e7500c81b4240ce619c0/coverage-7.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:478b5bcd63c2e1357c5c7e16c070690df7b07f676b1c114d7b93e533c664309f", size = 219848, upload-time = "2026-05-26T20:38:38.715Z" }, + { url = "https://files.pythonhosted.org/packages/91/82/a5eb47257c50601bb7b9a9d2857c67b7a3a85ad74180eb2c98bb1fbe0ce5/coverage-7.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a24a81f9715ee42ef59a316cc11611c98fe23920f7c81861315c9f3ff4a230f4", size = 220354, upload-time = "2026-05-26T20:38:40.232Z" }, + { url = "https://files.pythonhosted.org/packages/43/8b/78419b5391a5cb706b6544390507e469d83ffc9a8248b02c4011aceb9365/coverage-7.14.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:196a13319ad88d6d8ef5ab489ec4f44ddde2143c0c7d5b27786f6c3ffd56a7e1", size = 250771, upload-time = "2026-05-26T20:38:41.782Z" }, + { url = "https://files.pythonhosted.org/packages/77/63/e77aaacd491182210d639636b7a8bba23ffffa9b82aa3762da9431855fa9/coverage-7.14.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d452fd08b5c72c5167c93e6867b5c08500bd40f2a21e1e854a500550b6cc36f", size = 252683, upload-time = "2026-05-26T20:38:43.305Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/a022e3cfbec2ac241640003cb3a817e161d9c7f5aa9b49173756cdc03204/coverage-7.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23bf7fa51ac02e07fc7c96849b82946da47ae862dc8f86d183b2a4864fc38129", size = 254791, upload-time = "2026-05-26T20:38:45.361Z" }, + { url = "https://files.pythonhosted.org/packages/61/d6/967e408aca4c1ceb88cb0cc677169110ae7f5995fb5eaf5fb1f5a1bb8f5d/coverage-7.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcaa50684dcaadfa599ac48f81103c756d791cfd85c97203d2217c593d48b860", size = 256748, upload-time = "2026-05-26T20:38:46.91Z" }, + { url = "https://files.pythonhosted.org/packages/b8/be/869188f7fe28638078ec479331ace6dc5f7b40b7153eb616f47ab79404d8/coverage-7.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4ea1c034f95c9b056e856b794630b17f9fa3d57e4800ff1e503d3be0f9c9078c", size = 250907, upload-time = "2026-05-26T20:38:48.493Z" }, + { url = "https://files.pythonhosted.org/packages/07/aa/adb7d3b4278d690e68703abcd76ab1b948242e3668d921711551b78f9ddb/coverage-7.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e057326434e441306226fbeb5d1aaf14a2637efe97ba668306635835f32ad7", size = 252483, upload-time = "2026-05-26T20:38:50.074Z" }, + { url = "https://files.pythonhosted.org/packages/43/61/331c74103c62dcb0c4b9b3a0de9a61aca016208b0a90f109592a9f9ecc28/coverage-7.14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:59baf88468dbc8d63b1887afd92bda52e40bb1561696e5819670601403810cec", size = 250545, upload-time = "2026-05-26T20:38:51.613Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/c5dae3c104d89be04828f61810e6b3473825482e4c288cc4ed04553e08ae/coverage-7.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d34d75f892b3ab73ba11cab5442cce7b3e168fd64162b16f0e1e0d09c508edef", size = 254310, upload-time = "2026-05-26T20:38:53.503Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a1/2b9d5863e3b83c01ad8199e3c597802fbb3a9dc90b058885804c20296d31/coverage-7.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3a56abc20a472baf0304c455721bc601477440d28ecfde8a03dde79ede07e0df", size = 250266, upload-time = "2026-05-26T20:38:55.414Z" }, + { url = "https://files.pythonhosted.org/packages/7f/5e/0e511fbdb269359be26fe678a1c3fa1f2aa2a01573cc3f54268c8d6d4797/coverage-7.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a3cb83d1552c0cd1b4906655b6a33fd4a8473229633a901c6b73bf86914dee9", size = 251174, upload-time = "2026-05-26T20:38:57.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/10/e55307b622b3dd9671cb321824502dc10f93e72f2802b9946159a8edadeb/coverage-7.14.1-cp311-cp311-win32.whl", hash = "sha256:10274a1fbeb8ec5d72966e17bb198a3104257aca4ac09d98667c5f8aca8c8548", size = 222354, upload-time = "2026-05-26T20:38:58.727Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/107421693cfb71e4f1ca5bf70443f64d4161878068d07a3e51c7ad21d17b/coverage-7.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:87ebdf787d4888e3f3f2d523eadc6e18c6d18c6d0eb173801a189641627fb37e", size = 223290, upload-time = "2026-05-26T20:39:00.413Z" }, + { url = "https://files.pythonhosted.org/packages/b8/1d/3e3644585eb29e9dafefb19555078529a4d7cce12bd21929664eea989277/coverage-7.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:dd34767fa19848d35659ffc0a75314f58c7af3f1cd87ec521e8292a1238398a3", size = 221953, upload-time = "2026-05-26T20:39:02.159Z" }, + { url = "https://files.pythonhosted.org/packages/3d/b7/bdbb725ba02c5b42825b200c940f38b7a54fcad24627b7192f78f8110d76/coverage-7.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a06c76364a9360e33d6d23769aefdf7f66f38e2ffb60ceb1baaa4989d83b695c", size = 220022, upload-time = "2026-05-26T20:39:03.702Z" }, + { url = "https://files.pythonhosted.org/packages/72/81/fdc0898a55c6219223291ec1a1fe89966ef212ce82276aa0899df84b5de0/coverage-7.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fad54e871165f6ec2f536063ac74c3104508a12963e64072ba44bd822de52b0c", size = 220379, upload-time = "2026-05-26T20:39:05.381Z" }, + { url = "https://files.pythonhosted.org/packages/de/72/de048c4a25e13bce59ac6a339351c10bdf2515e07459afcdaf04dc3143a2/coverage-7.14.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:84b535f00655ecafe1d929d1fb00ed5d6fa3051ea643ab2c161a3887b86f294b", size = 251888, upload-time = "2026-05-26T20:39:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/28/30/300c343f68beb9d4cbb64ec81e58c5b6b80b56927f72d2b38654ac26e013/coverage-7.14.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6b6b0853b895fe0e98cbfc580d1ec3393d9302b4b1e96a77b3f5c91fdab899e6", size = 254624, upload-time = "2026-05-26T20:39:09.037Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ed/7b25642496e8170b6bac14adce00537c6e5fa2d586159401a4de3e8b49e6/coverage-7.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:442cc9c952b2df400cda54bb04ab87330cf2cd08a8692cbbea36773531eb6f37", size = 255739, upload-time = "2026-05-26T20:39:10.889Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a2/abd210b8c4e29c24e4624916db97bb519097a91034aaeb767f937e7da794/coverage-7.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8270544c361ed405a27a060dbc9ed2c124b084d96dfdc2d9a2510482aef981ad", size = 257998, upload-time = "2026-05-26T20:39:12.722Z" }, + { url = "https://files.pythonhosted.org/packages/7f/24/7c50beed3792fe62f6ce0545c6686ce83379719e2c0276179333d97eae92/coverage-7.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:48b283b1dd6372e8de2a7a9a4c4d5dc06f4d4fd209b876f3c88a7a205a0c8f84", size = 252296, upload-time = "2026-05-26T20:39:14.259Z" }, + { url = "https://files.pythonhosted.org/packages/15/05/0f874628ebcbfc77ead559ff210281ef06a97db08481832e7dd39274a135/coverage-7.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5b0c99ba93a07d56f6df340bb79be53202a082b2fdb81bfe6190b741a3470d54", size = 253658, upload-time = "2026-05-26T20:39:15.923Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/ca6ad067364b337ef997802115e7ecad2abd2248b05471464b0dea02b4d4/coverage-7.14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e471bc5769ff073b058cfadb0d736b56ce067c8560eabeb0da88462df98c23e7", size = 251803, upload-time = "2026-05-26T20:39:17.537Z" }, + { url = "https://files.pythonhosted.org/packages/c0/30/b9b4d377cd9f40baf228068f5a81faf8450c6228503011bd499708483a50/coverage-7.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f497a1ea81d4cd7c10ddcaa685135b9aabd291af3d55775a9ddf3cb7a364cdd9", size = 255873, upload-time = "2026-05-26T20:39:19.414Z" }, + { url = "https://files.pythonhosted.org/packages/3c/21/7c721a9e5e6bb88547d30a787aefb97512d3f54c1324c7488d9b3743f7f9/coverage-7.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2222be86d0b54f5dd5a38f45f17f315f737245e857bf0bdedc70734f84a13c02", size = 251372, upload-time = "2026-05-26T20:39:21.169Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f8ae5a2200130e1503cd7661a6cd3b2b7bacef98277fbf3571fb13f8b766/coverage-7.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85e85586565842f6932abebd4c18bcb1074223dc0b3576e7d173ca710622813a", size = 253245, upload-time = "2026-05-26T20:39:23.097Z" }, + { url = "https://files.pythonhosted.org/packages/34/62/70a9024672a5f6910517d9628c52c9afbdd3cf8f46426af52bb148a56fff/coverage-7.14.1-cp312-cp312-win32.whl", hash = "sha256:4a28fd227808366b196a75476dced2eb35b351d6766ba9c858dc93319e87f4f1", size = 222567, upload-time = "2026-05-26T20:39:24.868Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/8b7cd386839b039ebe1855733b9f9449a8dec5d79564018234f185a7fa70/coverage-7.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:54acdb6674a4661768d7bf7db32dfb9f46ab1d764f8aba6df75ce1a6a088724e", size = 223372, upload-time = "2026-05-26T20:39:26.603Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ba/b44d472022f620d289d95fa830143235c0c36461c6f2437ea8d51e5481ed/coverage-7.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:99cd41ff91afd94896fea3bc002706b6ae4ce95727d06e4a0f39c0a8d8bd8b1a", size = 221989, upload-time = "2026-05-26T20:39:28.242Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9e/5f6d56327c62b185225d145191c607e07515294a0aa6338e58805cd4a5ac/coverage-7.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793", size = 220044, upload-time = "2026-05-26T20:39:29.902Z" }, + { url = "https://files.pythonhosted.org/packages/75/92/e82aca356744cbbc0f77a0b623e38918c1872361963413a3bab5d0340393/coverage-7.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d", size = 220412, upload-time = "2026-05-26T20:39:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/27/c9/385bde0bf7ed0f4bf3a7ee5367060a86b5d218718cfd6fb943c0f836b34f/coverage-7.14.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247", size = 251412, upload-time = "2026-05-26T20:39:33.337Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/23faf6a2343a0d17f960a4bd56c43bc7eb4cf312f774dd6ceebd82c7d8fc/coverage-7.14.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d", size = 254008, upload-time = "2026-05-26T20:39:35.009Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/36f4aa9ca8a815e6036156e80706a67828bb97bd826948244f6996dda957/coverage-7.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b", size = 255241, upload-time = "2026-05-26T20:39:36.71Z" }, + { url = "https://files.pythonhosted.org/packages/ca/79/95266316352f90f6b1c6736bb413302edfde2453fb32422d3911642691b3/coverage-7.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be", size = 257373, upload-time = "2026-05-26T20:39:38.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/9c/58316d1f66c488b5fca8a0eb3e98348807813efa8a0d0833b9021be27488/coverage-7.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43", size = 251635, upload-time = "2026-05-26T20:39:40.268Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5a/ca2398a568e16fed7bb713e84ba3603a7164fb65779abe645c565ec890d5/coverage-7.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901", size = 253373, upload-time = "2026-05-26T20:39:42.145Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2c/0396562c32deaebe7be51d865b3a41e9a87d7561acafe1a28f53b07e019a/coverage-7.14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff", size = 251341, upload-time = "2026-05-26T20:39:43.907Z" }, + { url = "https://files.pythonhosted.org/packages/fd/8f/a94f9221184c9cae1ee115820e3798e48b6b17777a9f19e46fb9a0c8dc74/coverage-7.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4", size = 255497, upload-time = "2026-05-26T20:39:46.166Z" }, + { url = "https://files.pythonhosted.org/packages/71/69/505d70e47db1eaebcd002c39759707621ef184cd6b1ae084d9f41293f323/coverage-7.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d", size = 251159, upload-time = "2026-05-26T20:39:48.03Z" }, + { url = "https://files.pythonhosted.org/packages/e0/aa/58681c383aa33a9d2ed40a02d7a22fbf780d1fa4d575396365777828198c/coverage-7.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33", size = 252934, upload-time = "2026-05-26T20:39:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/eb/fd/11c928cd6bdffc7074bb5965c173d9ebf517fb00205e1da524b98d29ef92/coverage-7.14.1-cp313-cp313-win32.whl", hash = "sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c", size = 222584, upload-time = "2026-05-26T20:39:51.68Z" }, + { url = "https://files.pythonhosted.org/packages/6f/92/fb416fc26d340dcba19518c418d6048e913186e17243982c5e435e41fa7a/coverage-7.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416", size = 223394, upload-time = "2026-05-26T20:39:53.472Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/02d56e3867972f77d5036de924643f26c056e848f00452cafb4dbc3c29b4/coverage-7.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42", size = 222015, upload-time = "2026-05-26T20:39:55.374Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9e/fcc77914050df73f7662fa1f00902774c79c075a8388ab334074574bf77e/coverage-7.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d", size = 220733, upload-time = "2026-05-26T20:39:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/f7/67/2963cbdaf5cbadec44efa3a1e39eaa1f02df4079585f05387607a221e126/coverage-7.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5", size = 221086, upload-time = "2026-05-26T20:39:59.019Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c5/8701645574e11881f2f47d8930f98bc48b5d43b25eb5b4430dfc4a2f9f48/coverage-7.14.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52", size = 262381, upload-time = "2026-05-26T20:40:00.822Z" }, + { url = "https://files.pythonhosted.org/packages/7c/28/7a64d73598263e0c5abd5084211a8474488d31b3c552ff531c719dfcff62/coverage-7.14.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a", size = 264458, upload-time = "2026-05-26T20:40:02.506Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d8/4969179db9f7eb4df218e69540adf829d1c835f59452513d065d15446802/coverage-7.14.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a", size = 266884, upload-time = "2026-05-26T20:40:04.421Z" }, + { url = "https://files.pythonhosted.org/packages/a6/78/a45d5794dbc9bafd97afc96a4377c86c7820d78b6cf51b89bc1d4e919275/coverage-7.14.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2", size = 268022, upload-time = "2026-05-26T20:40:06.298Z" }, + { url = "https://files.pythonhosted.org/packages/21/cb/4f5e354e9e3e67af96bd4e57113e6db6b22298c7168b13eec408a549903d/coverage-7.14.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e", size = 261631, upload-time = "2026-05-26T20:40:08.226Z" }, + { url = "https://files.pythonhosted.org/packages/ec/49/eced49af4cb996d5d8b7e94e736175c513e4facd3398507b89892b4326d8/coverage-7.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d", size = 264443, upload-time = "2026-05-26T20:40:10.137Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d8/5603a88a7c5913a6b54f6cb1a8c46f7b39cbb30f27cd3f492908da09b2d7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb", size = 262069, upload-time = "2026-05-26T20:40:11.999Z" }, + { url = "https://files.pythonhosted.org/packages/f0/59/2ae3cb79da554a06c8619d6c88ea19dd1e4aed4b834b6a83bb1fa243bdc5/coverage-7.14.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d", size = 265780, upload-time = "2026-05-26T20:40:13.858Z" }, + { url = "https://files.pythonhosted.org/packages/af/5f/b130c1dc999031f2648bd25317fbce505ad8d5562079b4ed81e736a84967/coverage-7.14.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69", size = 260970, upload-time = "2026-05-26T20:40:16.142Z" }, + { url = "https://files.pythonhosted.org/packages/87/d1/ec13ccddeb48ec963bdfa72a11224bac2584bd045ba13beca82f8113e9c7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54", size = 263157, upload-time = "2026-05-26T20:40:18.382Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c2/cd91ead503045161092d3845f7bb95ea2f25131ce96d3e314dd835d91b9c/coverage-7.14.1-cp313-cp313t-win32.whl", hash = "sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1", size = 223259, upload-time = "2026-05-26T20:40:20.381Z" }, + { url = "https://files.pythonhosted.org/packages/71/9f/1e28d97e6bd2c76b07f38b7c02870f1371255ff6717f54eca578fcbbdd0e/coverage-7.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce", size = 224320, upload-time = "2026-05-26T20:40:22.316Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e0/d936e908f0e1efa55e52b91e01b52f1055cef5e1ab2718493390ed8e2fb8/coverage-7.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1", size = 222577, upload-time = "2026-05-26T20:40:24.894Z" }, + { url = "https://files.pythonhosted.org/packages/d6/34/fc2f101b151af3799a101f0550b0454aa008afdc0add677394ec4aa8ea10/coverage-7.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee", size = 220091, upload-time = "2026-05-26T20:40:27.249Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a7/1ebae2ab5b961b5c79bb09fe7b3ac99edb190d8be4a8c510b2cf66f46468/coverage-7.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500", size = 220421, upload-time = "2026-05-26T20:40:30.084Z" }, + { url = "https://files.pythonhosted.org/packages/5e/90/92aca9cf0acc95123c96cd1eb1f08917897a7f5dee01e15738922971ec31/coverage-7.14.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906", size = 251466, upload-time = "2026-05-26T20:40:32.542Z" }, + { url = "https://files.pythonhosted.org/packages/26/2b/78048cbe3b999f6cbf9cc0d90abba6a88a3e0863a8c1c6cbc762f3f8802f/coverage-7.14.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42", size = 253973, upload-time = "2026-05-26T20:40:34.473Z" }, + { url = "https://files.pythonhosted.org/packages/8e/21/c2e33b29d1cfde484a19d437afc343c6cd30b08d78cbbf9f5aff14e57b2b/coverage-7.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8", size = 255318, upload-time = "2026-05-26T20:40:38.154Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ee/aad2f108d63b769121005302f16bf66db8625c88ceaba466942e09a2607e/coverage-7.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851", size = 257633, upload-time = "2026-05-26T20:40:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f8/11a2c29b4fd76d9849f81d0bb812ec0017a9396df3217214e38934a8c837/coverage-7.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034", size = 251488, upload-time = "2026-05-26T20:40:42.631Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b8/9a5820de4b8ac2b71d85e3b5fb49108d7469c665f0e2ad0dd7569023e305/coverage-7.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c", size = 253329, upload-time = "2026-05-26T20:40:45.208Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ff/f33e4823667e27548e8fd8df44217515303f9808d0ff29817db56f87d990/coverage-7.14.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36", size = 251291, upload-time = "2026-05-26T20:40:47.502Z" }, + { url = "https://files.pythonhosted.org/packages/68/9b/489db0ebb209054766b90a9014a45f6d26eb724c02ec21311c3733b5a644/coverage-7.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5", size = 255564, upload-time = "2026-05-26T20:40:49.372Z" }, + { url = "https://files.pythonhosted.org/packages/27/b5/16bc2d4c2409b23c7737edb68c83bc89e345f378050549fe1d75ac7d34d5/coverage-7.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4", size = 251107, upload-time = "2026-05-26T20:40:51.677Z" }, + { url = "https://files.pythonhosted.org/packages/7d/0c/2629997469a00cd069d588a41c9dc887610f2775ae89d250c4791e65272a/coverage-7.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d", size = 252764, upload-time = "2026-05-26T20:40:54.267Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ee/f78d63c8f079e0d7211c7e2401fa17e311514534ba61bae03e4b287ce4ab/coverage-7.14.1-cp314-cp314-win32.whl", hash = "sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee", size = 222837, upload-time = "2026-05-26T20:40:56.496Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b9/be539854f93a70dfbeec69117f33ec70dc42ff0b65b5b07ab8d40d04228e/coverage-7.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7", size = 223650, upload-time = "2026-05-26T20:40:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9e/24e2842fef40f35ac82ba3a7719c8023d011bf3bf652d0675316a9d088a1/coverage-7.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343", size = 222218, upload-time = "2026-05-26T20:41:00.321Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1d/ac0a9df5fe31c1e8bdd658074905fc12844a05c1a7e3fdb8417e97c31e23/coverage-7.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1", size = 220822, upload-time = "2026-05-26T20:41:02.281Z" }, + { url = "https://files.pythonhosted.org/packages/32/cf/f964fd9aff20323f9f1a726c97135f8a76bcd87b92dad141a456a43f3c64/coverage-7.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b", size = 221084, upload-time = "2026-05-26T20:41:04.593Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5e/7e5ef2aba844de2b80d678619fcf0841b42e3f37f16411226f3fe4c1016f/coverage-7.14.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474", size = 262454, upload-time = "2026-05-26T20:41:06.641Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/75809bded87015cc4935524218a2a8ed8dd1a8498bfed30a2f4f7a4b4d34/coverage-7.14.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86", size = 264578, upload-time = "2026-05-26T20:41:08.556Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/d33392dc14633525012d2d504fa1a33b05538bf535f5c1d64675e5754b78/coverage-7.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e", size = 266981, upload-time = "2026-05-26T20:41:10.824Z" }, + { url = "https://files.pythonhosted.org/packages/2a/49/0157c4428c2aca7f1e09d5565930586fd5ae36f1655f08b0daa7cf1fcae1/coverage-7.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65", size = 268112, upload-time = "2026-05-26T20:41:12.966Z" }, + { url = "https://files.pythonhosted.org/packages/96/26/86b9ce71f4092b1ed325ce1421698081df1286b833400b6836912834d6e0/coverage-7.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e", size = 261558, upload-time = "2026-05-26T20:41:15Z" }, + { url = "https://files.pythonhosted.org/packages/20/4c/c311210c5472cf5401d8422b0d7812cdd520f24417673afabda6c323faca/coverage-7.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8", size = 264447, upload-time = "2026-05-26T20:41:17.369Z" }, + { url = "https://files.pythonhosted.org/packages/fb/71/59513f8710ed3e6b0ac0a050a5b7e977bb9c9e880354863b5d00d8809256/coverage-7.14.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07", size = 262048, upload-time = "2026-05-26T20:41:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/84/8d/bceed32dc494f5bbf50f775cd2e78ca814953942b5ea28d3c1c3ac316f14/coverage-7.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de", size = 265781, upload-time = "2026-05-26T20:41:21.559Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c5/9348fe40dbfd4991aaf78df2c6c3098bfb2cc834d1fd362a64b4efef855a/coverage-7.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890", size = 260896, upload-time = "2026-05-26T20:41:23.428Z" }, + { url = "https://files.pythonhosted.org/packages/ca/92/1ea0f03929da7cf87206b1fa24f4c8e9c158be0455481af29ec0a1f3503f/coverage-7.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd", size = 263214, upload-time = "2026-05-26T20:41:25.419Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a9/b2493c054c0e01a643266742ab45e15744e60743f9260cd930c7142b1124/coverage-7.14.1-cp314-cp314t-win32.whl", hash = "sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e", size = 223624, upload-time = "2026-05-26T20:41:27.795Z" }, + { url = "https://files.pythonhosted.org/packages/fc/bd/3e1e6a57fccd2d7c83fcdf338e93ba98eb85c6e877dd34731ac585375490/coverage-7.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c", size = 224728, upload-time = "2026-05-26T20:41:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d7/31066cf1d2f0c6c797fce911bcfa01dd35642dc6da992a950256097c5860/coverage-7.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af", size = 222752, upload-time = "2026-05-26T20:41:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + [[package]] name = "hve-telemetry" version = "0.0.0" @@ -30,6 +134,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ { name = "pytest" }, + { name = "pytest-cov" }, { name = "ruff" }, ] fuzz = [ @@ -41,6 +146,7 @@ fuzz = [ [package.metadata.requires-dev] dev = [ { name = "pytest", specifier = ">=9.0" }, + { name = "pytest-cov", specifier = ">=7.0" }, { name = "ruff", specifier = ">=0.15" }, ] fuzz = [{ name = "atheris", specifier = ">=3.0" }] @@ -97,6 +203,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "ruff" version = "0.15.16" @@ -121,3 +241,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/9e/02e845ef151b1dee585e55c4739f8e1734ae1d9f1221dff65761c162208b/ruff-0.15.16-py3-none-win_amd64.whl", hash = "sha256:408256017284eddf98fff77b29aa4fb30f586042d535b2d9befc6512f400aaec", size = 11843403, upload-time = "2026-06-04T16:32:39.76Z" }, { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, ] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] From f36078f28f6fdaeb772a24b14caf0f203badb204 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 18 Jun 2026 17:51:46 -0600 Subject: [PATCH 07/10] add low risk security note --- docs/customization/local-telemetry.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/customization/local-telemetry.md b/docs/customization/local-telemetry.md index b806b21e0..db33d3789 100644 --- a/docs/customization/local-telemetry.md +++ b/docs/customization/local-telemetry.md @@ -3,7 +3,7 @@ title: Local Telemetry description: Enable local Copilot session telemetry, understand capture mechanics, and generate local reports sidebar_position: 10 author: Microsoft -ms.date: 2026-06-17 +ms.date: 2026-06-18 ms.topic: how-to keywords: - telemetry @@ -245,6 +245,20 @@ is required. Stale directories (deleted or moved repositories) are pruned automatically when the report runs. Each session is labeled with its originating project in the report, so combined output still reads per project. +> [!NOTE] +> **Registry-driven cleanup is name-constrained.** `clean-telemetry.sh +> --all-dirs` iterates every path in `~/.hve/telemetry-dirs.txt` and, in each +> directory, removes only a fixed allow-list of artifact names +> (`raw-input.jsonl`, `report.generated.html`, `sessions-*.jsonl`, and the +> `.stacks/` directory). It never deletes a directory wholesale. A tampered +> registry can therefore, at most, delete those specific names in an +> attacker-chosen directory — not arbitrary files. The `.stacks/` entry is +> removed recursively, but symlinked artifacts are unlinked rather than +> followed, so the target of a symlink is never deleted. The registry lives in +> the user-owned HVE home (`~/.hve`, honoring `HVE_HOME`), so an attacker able +> to tamper with it already holds the user's filesystem privileges; the risk is +> low and the blast radius is bounded. + ## Reports Without the Repository (Extension Users) When telemetry runs from the VS Code extension rather than this repository, the From 393c92115f4c65faa6627de36f3493647301c5d9 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 18 Jun 2026 17:54:22 -0600 Subject: [PATCH 08/10] fix lint --- docs/customization/local-telemetry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/customization/local-telemetry.md b/docs/customization/local-telemetry.md index db33d3789..54bb36cc4 100644 --- a/docs/customization/local-telemetry.md +++ b/docs/customization/local-telemetry.md @@ -252,7 +252,7 @@ project in the report, so combined output still reads per project. > (`raw-input.jsonl`, `report.generated.html`, `sessions-*.jsonl`, and the > `.stacks/` directory). It never deletes a directory wholesale. A tampered > registry can therefore, at most, delete those specific names in an -> attacker-chosen directory — not arbitrary files. The `.stacks/` entry is +> attacker-chosen directory, not arbitrary files. The `.stacks/` entry is > removed recursively, but symlinked artifacts are unlinked rather than > followed, so the target of a symlink is never deleted. The registry lives in > the user-owned HVE home (`~/.hve`, honoring `HVE_HOME`), so an attacker able From 8e3487a111f73e99a45413cd848476d6748a7513 Mon Sep 17 00:00:00 2001 From: Vy Ta Date: Thu, 25 Jun 2026 14:07:24 -0700 Subject: [PATCH 09/10] comments for passthroughs, experimental maturity marker, HVE_TELEMETRY_RAW parity --- .../telemetry/Invoke-TelemetryCollector.ps1 | 18 ++++++++++++++++++ .../hooks/shared/telemetry/_telemetry_core.py | 10 ++++++++++ collections/hve-core-all.collection.yml | 1 + collections/hve-core.collection.yml | 1 + 4 files changed, 30 insertions(+) diff --git a/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 index 8f6b12f52..4be279d84 100644 --- a/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 +++ b/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 @@ -68,6 +68,24 @@ if (-not (Test-Path $TelemetryDir)) { # Delegate all JSON processing to the shared Python telemetry engine $RawInput = $input | Out-String + +# Dump raw input for diagnostics (first 5 events only). This records hook +# payloads verbatim, including the full prompt text and tool inputs such as +# file contents and shell command strings, which can contain secrets. The +# processed sessions-*.jsonl stream already provides the diagnostic signal, +# so the verbatim dump is a separate explicit opt-in (off by default) layered +# on top of the telemetry gate. See docs/customization/local-telemetry.md. +if ($env:HVE_TELEMETRY_RAW -eq '1') { + $RawLog = Join-Path $TelemetryDir 'raw-input.jsonl' + $RawCount = 0 + if (Test-Path $RawLog) { + $RawCount = (Get-Content -LiteralPath $RawLog).Count + } + if ($RawCount -lt 5) { + Add-Content -LiteralPath $RawLog -Value $RawInput + } +} + try { $env:HVE_REPO_ROOT = $RepoRoot $env:HVE_TELEMETRY_DIR = $TelemetryDir diff --git a/.github/hooks/shared/telemetry/_telemetry_core.py b/.github/hooks/shared/telemetry/_telemetry_core.py index 561cf4edf..65def4d36 100644 --- a/.github/hooks/shared/telemetry/_telemetry_core.py +++ b/.github/hooks/shared/telemetry/_telemetry_core.py @@ -113,6 +113,7 @@ def read_registry_dirs(registry: Path | None = None) -> list[str]: try: text = registry.read_text(encoding="utf-8") except OSError: + # Registry file is missing or unreadable; treat as no registered dirs. return [] dirs: list[str] = [] seen: set[str] = set() @@ -143,6 +144,7 @@ def register_telemetry_dir(tel_dir: Path, registry: Path | None = None) -> None: with open(registry, "a", encoding="utf-8") as handle: handle.write(resolved + "\n") except OSError: + # Cannot create or append to the registry; skip recording this dir. return @@ -270,6 +272,7 @@ def write_report_launchers(script_dir: Path | None = None) -> None: clean_sh_path.write_text(bash_clean_text, encoding="utf-8") clean_sh_path.chmod(0o755) except OSError: + # Cannot write launchers (e.g., permission denied); skip generation. return @@ -282,6 +285,7 @@ def find_process_log(state_dir: Path, home: Path) -> str | None: pid = lock_file.split(".")[1] break except OSError: + # State dir cannot be listed (e.g., does not exist); no log to find. return None if not pid: return None @@ -301,6 +305,7 @@ def _log_references_interactions(log_path: str, interaction_ids: set[str]) -> bo if "interaction_id" in line and any(iid in line for iid in interaction_ids): return True except OSError: + # Log cannot be read; treat as not referencing this session. return False return False @@ -362,6 +367,7 @@ def parse_process_log(log_path: str, interaction_ids: set[str]) -> list[dict]: in_block = False block_lines = [] except OSError: + # Log cannot be read; return whatever was parsed so far. return results return results @@ -697,6 +703,7 @@ def _token_estimate(path: str) -> int: # Ceiling division without importing math. return int(-(-os.path.getsize(path) // 4)) except OSError: + # File size unavailable (e.g., missing file); estimate zero tokens. return 0 @@ -719,6 +726,7 @@ def _read(self) -> list[str]: if isinstance(data, list): return data except (OSError, ValueError): + # Stack file is unreadable or malformed; treat as empty stack. return [] return [] @@ -970,6 +978,7 @@ def _remove_path(path: Path, dry_run: bool, removed: list[str]) -> None: else: path.unlink() except OSError: + # Removal failed (e.g., permission denied); leave the path in place. return removed.append(str(path)) @@ -1040,6 +1049,7 @@ def _mode_list_dirs() -> int: with open(registry, "w", encoding="utf-8") as handle: handle.write("".join(d + "\n" for d in live)) except OSError: + # Cannot rewrite the registry; keep stale entries rather than fail. pass for directory in live: sys.stdout.write(directory + "\n") diff --git a/collections/hve-core-all.collection.yml b/collections/hve-core-all.collection.yml index 169c4bc9a..5a1fdfb0f 100644 --- a/collections/hve-core-all.collection.yml +++ b/collections/hve-core-all.collection.yml @@ -601,6 +601,7 @@ items: kind: skill - path: .github/hooks/shared/telemetry.json kind: hook + maturity: experimental display: featured: true ordering: alpha diff --git a/collections/hve-core.collection.yml b/collections/hve-core.collection.yml index 6a4d6765d..2ed099112 100644 --- a/collections/hve-core.collection.yml +++ b/collections/hve-core.collection.yml @@ -152,5 +152,6 @@ items: # Hooks - path: .github/hooks/shared/telemetry.json kind: hook + maturity: experimental display: ordering: manual From d387669a0860b42d792e858e8eb06522af298e61 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 26 Jun 2026 19:03:55 -0700 Subject: [PATCH 10/10] fix(hooks): normalize telemetry copyright headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - update copyright header text to canonical format across telemetry hook scripts - strip erroneous UTF-8 BOM added to shebang .sh/.py scripts 🔒 - Generated by Copilot --- .github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 | 2 +- .github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 | 2 +- .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 | 2 +- .github/hooks/shared/telemetry/_telemetry_core.py | 2 +- .github/hooks/shared/telemetry/clean-telemetry.sh | 2 +- .github/hooks/shared/telemetry/generate-telemetry-report.sh | 2 +- .github/hooks/shared/telemetry/telemetry-collector.sh | 2 +- .github/hooks/shared/telemetry/tests/fuzz_harness.py | 2 +- .github/hooks/shared/telemetry/tests/test_telemetry_core.py | 2 +- scripts/linting/Validate-HookManifests.ps1 | 2 +- scripts/tests/linting/Validate-HookManifests.Tests.ps1 | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 index 743b620e6..96bce77e4 100644 --- a/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 +++ b/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 @@ -1,5 +1,5 @@ #!/usr/bin/env pwsh -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT #Requires -Version 7.0 diff --git a/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 index 4be279d84..1e97bf9c8 100644 --- a/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 +++ b/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 @@ -1,5 +1,5 @@ #!/usr/bin/env pwsh -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT #Requires -Version 7.0 diff --git a/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 index c31e8a52c..1fb8cc24e 100644 --- a/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 +++ b/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 @@ -1,5 +1,5 @@ #!/usr/bin/env pwsh -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT #Requires -Version 7.0 diff --git a/.github/hooks/shared/telemetry/_telemetry_core.py b/.github/hooks/shared/telemetry/_telemetry_core.py index 65def4d36..b90543c9d 100644 --- a/.github/hooks/shared/telemetry/_telemetry_core.py +++ b/.github/hooks/shared/telemetry/_telemetry_core.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT """Canonical telemetry engine shared by the Copilot hook collectors. diff --git a/.github/hooks/shared/telemetry/clean-telemetry.sh b/.github/hooks/shared/telemetry/clean-telemetry.sh index b5a938aeb..617b23db8 100755 --- a/.github/hooks/shared/telemetry/clean-telemetry.sh +++ b/.github/hooks/shared/telemetry/clean-telemetry.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT # # clean-telemetry.sh diff --git a/.github/hooks/shared/telemetry/generate-telemetry-report.sh b/.github/hooks/shared/telemetry/generate-telemetry-report.sh index b9979f8d7..6d38c5cbb 100755 --- a/.github/hooks/shared/telemetry/generate-telemetry-report.sh +++ b/.github/hooks/shared/telemetry/generate-telemetry-report.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT # # generate-telemetry-report.sh diff --git a/.github/hooks/shared/telemetry/telemetry-collector.sh b/.github/hooks/shared/telemetry/telemetry-collector.sh index de833dd38..e98dfb4f0 100755 --- a/.github/hooks/shared/telemetry/telemetry-collector.sh +++ b/.github/hooks/shared/telemetry/telemetry-collector.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT # # telemetry-collector.sh diff --git a/.github/hooks/shared/telemetry/tests/fuzz_harness.py b/.github/hooks/shared/telemetry/tests/fuzz_harness.py index 26d15270f..1df633a72 100644 --- a/.github/hooks/shared/telemetry/tests/fuzz_harness.py +++ b/.github/hooks/shared/telemetry/tests/fuzz_harness.py @@ -1,4 +1,4 @@ -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT """Polyglot fuzz harness for telemetry core logic. diff --git a/.github/hooks/shared/telemetry/tests/test_telemetry_core.py b/.github/hooks/shared/telemetry/tests/test_telemetry_core.py index 05ee35bd8..771195d55 100644 --- a/.github/hooks/shared/telemetry/tests/test_telemetry_core.py +++ b/.github/hooks/shared/telemetry/tests/test_telemetry_core.py @@ -1,4 +1,4 @@ -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT """Tests for the canonical telemetry engine (_telemetry_core).""" diff --git a/scripts/linting/Validate-HookManifests.ps1 b/scripts/linting/Validate-HookManifests.ps1 index e9dbf20e1..8cf655a4a 100644 --- a/scripts/linting/Validate-HookManifests.ps1 +++ b/scripts/linting/Validate-HookManifests.ps1 @@ -1,5 +1,5 @@ #!/usr/bin/env pwsh -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT #Requires -Version 7.0 diff --git a/scripts/tests/linting/Validate-HookManifests.Tests.ps1 b/scripts/tests/linting/Validate-HookManifests.Tests.ps1 index 629810e65..0d4793be9 100644 --- a/scripts/tests/linting/Validate-HookManifests.Tests.ps1 +++ b/scripts/tests/linting/Validate-HookManifests.Tests.ps1 @@ -1,5 +1,5 @@ #Requires -Modules Pester -# Copyright (c) Microsoft Corporation. +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT BeforeAll {