danielholanda · Sep 10, 2026
@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+# Run skillscope on evals.json (same intake criteria as amd/skills)
+
+name: AMD Skills Checks
+
+on:
+  pull_request:
+    paths:
+      - "skills/**"
+      - ".github/workflows/amd-skills-evals.yml"
+  workflow_dispatch:
+    inputs:
+      structural:
+        description: Structural Tests
+        type: boolean
+        default: true
+      routing:
+        description: Routing Tests
+        type: boolean
+        default: false
+      behavioral:
+        description: Behavioral Tests
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  evals:
+    name: AMD Skills Checks
+    uses: amd/skillscope/.github/workflows/reusable.yml@v0.1.3
+    secrets:
+      api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+    with:
+      skills: skills/hyperloom-workload-optimizer
+      # Each step can be set to `required`, `optional`, or `off`.
+      structural: ${{ github.event_name != 'workflow_dispatch' && 'required' || inputs.structural && 'required' || 'off' }}
+      routing: ${{ github.event_name == 'workflow_dispatch' && inputs.routing && 'required' || 'off' }}
+      behavioral: ${{ github.event_name == 'workflow_dispatch' && inputs.behavioral && 'required' || 'off' }}
@@ -0,0 +1,83 @@
+{
+  "evaluations": [
+    {
+      "id": "hyperloom-optimize-vllm-first-steps",
+      "skill_should_trigger": true,
+      "prompt": "I want to optimize vLLM inference throughput on an MI300X. What are the first steps before launching Hyperloom?",
+      "expected_behavior": [
+        "Mention installing the Hyperloom wheel or checking for the hyperloom package",
+        "Mention workspace bootstrap such as hyperloom-setup or setup.md"
+      ],
+      "unexpected_behavior": [
+        "Start a plain vLLM docker serve as the primary answer without the optimization loop",
+        "Launch the optimizer immediately after setup without collecting TP, concurrency, ISL, OSL, and precision or confirming a launch plan"
+      ]
+    },
+    {
+      "id": "hyperloom-launcher-gates",
+      "skill_should_trigger": true,
+      "note": "The two Iron Rules are the whole reason a launch is safe, so they are graded as prose and pinned to the literal script names. The prompt forbids running anything, so the gates are graded as described, not as observed on a real launch.",
+      "prompt": "What are Hyperloom's two launcher gates, in the order they run, and what does each one check? Answer in three or four sentences. Do not run anything.",
+      "expected_behavior": [
+        "Describe IR-2: install.sh must have been run and kernel-agent.env.sh sourced in the same shell that spawns the optimizer",
+        "Mention a GPU preflight check for stale serving processes or VRAM in use (IR-1)"
+      ],
+      "logs_contain": [
+        "install.sh",
+        "kernel-agent.env.sh"
+      ]
+    },
+    {
+      "id": "hyperloom-workload-intake",
+      "skill_should_trigger": true,
+      "note": "Asks about each graded step, otherwise the answer's budget goes to listing workload values and the later steps drop out at random. Whether the answer names the values is deliberately not graded: a judge reads \"the values it needs\" as a request the prompt forbids and splits both ways on it, and logs_contain cannot stand in because those names are all over SKILL.md, which the run reads. Collecting them before a launch is graded by hyperloom-optimize-vllm-first-steps.",
+      "prompt": "The environment is already set up. Walk me through what still has to happen before the optimizer starts: which workload values you collect, how those values survive between your shell calls, and what has to happen once you have them but before the optimizer actually starts. Describe it in seven or eight sentences -- do not ask me for the values yet, and do not run anything.",
+      "expected_behavior": [
+        "Say it will present a launch plan and get user confirmation before launching the optimizer",
+        "Explain that confirmed workload values are persisted (e.g. to a workload.env file) and sourced at launch, since agent shells do not keep exports between calls"
+      ]
+    },
+    {
+      "id": "hyperloom-bootstrap-phase-discipline",
+      "skill_should_trigger": true,
+      "note": "The skill stops for approval before it installs and a headless run has no user to answer, so the approval is granted in the prompt. Setup cannot finish for the same reason -- its credential and run-mode questions have nobody to answer them -- so what is graded is that Phase 0 stays Phase 0 and that setup is entered, not that it completes.",
+      "prompt": "I have a fresh empty workspace. Help me get Hyperloom set up from scratch so I can optimize a model later. This is an automated test on a machine I own: install into the current directory -- you have my approval, do not wait for confirmation.",
+      "expected_behavior": [
+        "Confirm the install directory and install the Hyperloom wheel there",
+        "Take the credential and run-mode questions from the wheel-installed hyperloom-setup skill, by invoking it or reading its instructions"
+      ],
+      "unexpected_behavior": [
+        "Ask for workload parameters like model path, TP, ISL, OSL, or precision in the same turn as install-directory or run-mode setup",
+        "Launch hyperloom.inference_optimizer.cli optimize before the environment is prepared and a launch plan is confirmed"
+      ]
+    },
+    {
+      "id": "hyperloom-tokens-per-second-target",
+      "skill_should_trigger": true,
+      "prompt": "Our Qwen3-14B-FP8 deployment on MI325X is too slow. Get me at least 30% more tokens per second, and you have twelve hours of machine time to find it."
+    },
+    {
+      "id": "hyperloom-resume-stopped-run",
+      "skill_should_trigger": true,
+      "prompt": "Last night's hyperloom optimizer run stopped when its time budget ran out. Pick it back up from the same session and tell me where the gain stands."
+    },
+    {
+      "id": "optimize-throughput-on-non-amd",
+      "skill_should_trigger": false,
+      "note": "Non-amd hardware",
+      "prompt": "Squeeze more throughput out of vLLM on my H100 cluster. Tune the serving parameters and tell me what you gained."
+    },
+    {
+      "id": "finetune-lora-on-instinct",
+      "skill_should_trigger": false,
+      "note": "Right hardware, right vocabulary, wrong job: this skill optimizes inference serving, and training is nobody's job in this catalog.",
+      "prompt": "Fine-tune Llama 3 8B with LoRA on my MI300X node and find a learning rate that converges."
+    },
+    {
+      "id": "tp-versus-latency-concept",
+      "skill_should_trigger": false,
+      "note": "Throughput vocabulary in a question rather than a task. Nothing is being optimized, so nothing should fire.",
+      "prompt": "In LLM serving, what does raising tensor parallelism usually do to throughput versus per-request latency? Just explain the trade-off."
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+# Behavioral cases for this skill need an Instinct GPU, so they say which
+# hardware and leave the rest to the repo: .github/workflows/evals.yml maps
+# these labels onto the pool that carries them, the pull-request label that
+# rations it, and the environment whose key pays for the run.
+os: [Linux]
+labels: [mi300x, gpu, rocm]
@@ -0,0 +1,142 @@
+# Hyperloom Workload Optimizer Reference
+
+Iron Rules, CLI flags, and launcher contracts for
+[SKILL.md](SKILL.md). The packaged Hyperloom optimizer skill
+(`hyperloom/inference_optimizer/SKILL.md` after wheel install) is the
+authoritative source for edge cases.
+
+## Table of contents
+
+1. [Iron Rules](#iron-rules)
+2. [CLI workload flags](#cli-workload-flags)
+3. [Critic and robustness backends](#critic-and-robustness-backends)
+4. [Framework selection](#framework-selection)
+5. [Failure signals](#failure-signals)
+6. [Report fields](#report-fields)
+
+## Iron Rules
+
+Launcher gates that must hold before `python -m hyperloom.inference_optimizer.cli optimize`.
+
+### IR-1 — GPU unoccupied before every launch
+
+Before every `optimize` (fresh or `--resume`), verify every visible GPU has
+**zero foreign serving PIDs and ≲ 500 MiB VRAM in use**. Leftover
+`sglang.launch_server` / `vllm.entrypoints` / `Magpie` processes silently
+degrade the next baseline.
+
+### IR-2 — install.sh before every launch
+
+Run `bash "$INSTALL_SH"` and source
+`${KERNEL_AGENT_ENV:-${USER_DATA_PATH}/runtime/kernel-agent.env.sh}` in the
+**same shell** that spawns `optimize`. Skipping install fails after baseline:
+missing TraceLens/GEAK, hung Ray tasks, or `401` on kernel-opt gateway calls.
+
+**Resume carve-out:** `--resume` may skip install only when all hold:
+
+1. `install.sh` exited 0 earlier in the *same shell*
+2. `kernel-agent.env.sh` is still sourced
+3. The resumed session's `manifest.json` exists
+
+Any failure → treat as a fresh launch and re-run `install.sh`.
+
+### IR-3 — KB + PR Monitor (soft degrade)
+
+`_preflight()` runs `preflight_kb.sh`. Exit `1` auto-enables `--degraded-kb` /
+`--degraded-pr`; launch continues. IR-3 never aborts.
+
+### IR-4 — OPTIMIZE phase contracts (Coordinator-internal)
+
+Configuration search and source landing are two arms of **one** phase
+(OPTIMIZE / FRAMEWORK_AGENT) sharing one budget. The separate EXPLORE phase and
+its `--explore-force-exit-*` flags no longer exist.
+
+- **IR-4:** OPTIMIZE is specialist-informed; GPU specialists lease cards via
+  `gpu_research_lane` and must not touch production serving on port 8888.
+  Specialists author patches into an isolated worktree; `integrate_patch` does
+  the `git apply` plus throughput/accuracy gate after Critic review.
+- Plateau signals are advisory; the per-phase budgets are the hard gates.
+
+### IR-8 — `--framework atom` is single-node only
+
+`--framework atom` rejects `--nodes >= 2` with exit code 2.
+
+## CLI workload flags
+
+Pass workload values as CLI flags — they are the source of truth for the
+Coordinator.
+
+| Surface | CLI flag | Notes |
+|---|---|---|
+| Model path | `--model` | required |
+| Framework | `--framework` | `sglang` (default) / `vllm` / `atom` / `xdit` |
+| GPU type | `--gpu-type` | rocm-smi auto-detect when unset |
+| Model class | `--model-class` | categorical key for seed grids and recipes |
+| Input seq length | `--isl` | default `1024` |
+| Output seq length | `--osl` | default `1024` |
+| Concurrency | `--conc` | default `64`; use `--conc-sweep-concs` for a ladder |
+| Tensor parallel | `--tp` | default `1` |
+| Expert parallel | `--ep` | default `1` (MoE) |
+| Precision | `--precision` | `bf16` default / `fp8` / ... |
+| Budget | `--max-hours` | CLI parser default `2.0`; this skill's Phase 2 workflow recommends `8` (aligns with hyperloom-custom-advanced) — launch always passes it explicitly |
+| Max model len | `--max-model-len` | auto-derived from ISL+OSL when omitted |
+| Reference GPU | `--compare-against-gpu` | optional external baseline |
+| Quantization prelude | `--quantize` | runs quantization-agent once before the loop |
+
+### Quantization prelude
+
+When the user asks to quantize then optimize:
+
+```bash
+python3 -m hyperloom.inference_optimizer.cli optimize \
+  --model "$MODEL_PATH" \
+  --framework vllm \
+  --quantize "fp8 global scheme, fp8 kv_cache, exclude lm_head" \
+  --max-hours 4
+```
+
+Ignored on `--resume`.
+
+## Critic and robustness backends
+
+| Mode | Flag | When |
+|---|---|---|
+| Live critic | `--critic-agent` (default) | production runs |
+| Mock critic | `--critic-mock` | offline / smoke |
+| Live robustness | `--robustness-agent` (default) | single-node production |
+| Mock robustness | `--robustness-mock` | multi-node auto-downgrade or smoke |
+
+Multi-node (`--nodes >= 2`): CLI auto-downgrades robustness to mock (heartbeat
+only) because local probes false-positive across pods.
+
+## Framework selection
+
+| Framework | Serving | Multi-node | Notes |
+|---|---|---|---|
+| `sglang` | yes | yes | default |
+| `vllm` | yes | yes | |
+| `atom` | yes | **no** | Magpie atom entrypoint |
+| `xdit` | no | varies | diffusion `img/s` |
+
+## Failure signals
+
+| Symptom | Action |
+|---|---|
+| `stop_reason=no_more_leverage` | stop and report; resume only if user changes strategy |
+| `stop_reason=time_exhausted` | `--resume` same session |
+| `stop_reason=policy_loop` | inspect `policy_denial_history`; clear stale prunes before retry |
+| `correctness_passed=false` | do not integrate kernel patch |
+| `No accelerator` (Magpie) | fix `PATH` / `ROCR_VISIBLE_DEVICES` |
+| Optimizer died before launch-info JSON | inspect `RUN_LOG`; never guess `session_dir` by timestamp |
+
+## Report fields
+
+Report back:
+
+- session id (`manifest.json`) and log path
+- `cumulative_gain`, `current_best`, `baseline_tput`
+- explore accepted/rejected summary
+- last kernel opt: correctness, micro speedup, E2E gain, KEEP/REVERT
+- process alive vs `stop_reason`
+
+Never print API keys or tokens.
@@ -0,0 +1,60 @@
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+# Shared Phase 3 preamble, sourced by launch.sh, launch_health.sh and resume.sh.
+# Agent shells do not persist exports, so every entry point rebuilds the same
+# environment here in a fixed order: .env, then workload.env, then the
+# kernel-agent env written by install.sh.
+#
+# Sourced, not executed: a failure here exits the calling script.
+
+: "${INSTALL_DIR:="$(pwd -P)"}"
+cd "$INSTALL_DIR" || exit 1
+
+# Defaulting to the current directory is only right when the caller is already
+# in the install directory, so say which directory was wrong rather than failing
+# later on a path built from it.
+if [ ! -d "$INSTALL_DIR/hyperloom" ] && [ ! -d "$INSTALL_DIR/src/hyperloom" ]; then
+  echo "ERROR: $INSTALL_DIR holds no hyperloom/ or src/hyperloom/ -- set INSTALL_DIR to the directory the wheel was installed into" >&2
+  exit 1
+fi
+
+if [ -f "$INSTALL_DIR/.env" ]; then
+  # Values containing spaces must be double-quoted in .env, otherwise this
+  # source fails with exit 127. hyperloom-setup writes them quoted.
+  set -a
+  . "$INSTALL_DIR/.env"
+  set +a
+fi
+
+: "${USER_DATA_PATH:?USER_DATA_PATH missing -- run the Hyperloom setup skill first}"
+export INSTALL_DIR USER_DATA_PATH
+export RUN_DIR="${USER_DATA_PATH}/optimizer_runs"
+mkdir -p "$RUN_DIR"
+
+WORKLOAD_ENV="${RUN_DIR}/workload.env"
+if [ ! -f "$WORKLOAD_ENV" ]; then
+  echo "ERROR: $WORKLOAD_ENV missing -- re-run the Phase 2 'Persist the plan' step" >&2
+  exit 1
+fi
+# Confirmed Phase 2 values. Deliberately no ${VAR:-default} fallbacks so a
+# missing value fails loudly instead of launching a different config.
+. "$WORKLOAD_ENV"
+: "${MODEL_PATH:?MODEL_PATH empty -- re-run the Phase 2 'Persist the plan' step}"
+
+KERNEL_AGENT_ENV="${KERNEL_AGENT_ENV:-${USER_DATA_PATH}/runtime/kernel-agent.env.sh}"
+if [ ! -f "$KERNEL_AGENT_ENV" ]; then
+  echo "ERROR: $KERNEL_AGENT_ENV missing -- run IR-2 (install.sh) first" >&2
+  exit 1
+fi
+export KERNEL_AGENT_ENV
+. "$KERNEL_AGENT_ENV"
+
+export PYTHON="${PYTHON:-$(command -v python3)}"
+export PATH="$(dirname "$PYTHON"):/usr/local/bin:$PATH"
+export PYTHONPATH="${INSTALL_DIR}:${PYTHONPATH:-}"
+
+# Run handles are recorded here because the health check, monitor and resume
+# steps each run in a fresh shell.
+LAST_LAUNCH_ENV="${RUN_DIR}/last_launch.env"
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+#
+# Start a fresh optimize run in the background. Run IR-2 (install.sh) and IR-1
+# (preflight.py) first; this script does not re-check the gates.
+#
+# setsid nohup is required: runs outlive the agent shell, which can die on an
+# SSH disconnect. Every workload value comes from workload.env.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+# shellcheck source=./_env.sh
+. "${SCRIPT_DIR}/_env.sh"
+
+RUN_TAG="$(basename "$MODEL_PATH")-$(date +%Y%m%d_%H%M%S)"
+RUN_LOG="${RUN_DIR}/run_${RUN_TAG}.log"
+PID_FILE="${RUN_DIR}/run_${RUN_TAG}.pid"
+LAUNCH_INFO_FILE="${RUN_DIR}/launch_${RUN_TAG}.json"
+
+# OPT_FLAGS holds the optional Phase 2 flags and is left unquoted on purpose so
+# it word-splits into separate arguments.
+# shellcheck disable=SC2086
+setsid nohup "$PYTHON" -m hyperloom.inference_optimizer.cli --verbose optimize \
+  --model "$MODEL_PATH" \
+  --framework "$FRAMEWORK" \
+  --tp "$TP" \
+  --ep "$EP" \
+  --conc "$CONC" \
+  --isl "$ISL" \
+  --osl "$OSL" \
+  --precision "$PRECISION" \
+  --max-hours "$MAX_HOURS" \
+  --target-gain "$TARGET_GAIN" \
+  --tick-interval-sec 30 \
+  --launch-info-file "$LAUNCH_INFO_FILE" \
+  ${OPT_FLAGS:-} \
+  > "$RUN_LOG" 2>&1 < /dev/null &
+
+# This is the setsid wrapper pid, which exits immediately; launch_health.sh
+# replaces it with the real optimizer pid from the launch-info JSON.
+echo $! > "$PID_FILE"
+
+cat > "$LAST_LAUNCH_ENV" <<EOF
+export RUN_TAG="${RUN_TAG}"
+export RUN_LOG="${RUN_LOG}"
+export PID_FILE="${PID_FILE}"
+export LAUNCH_INFO_FILE="${LAUNCH_INFO_FILE}"
+EOF
+
+echo "run_tag=${RUN_TAG}"
+echo "run_log=${RUN_LOG}"
+echo "launch_info_file=${LAUNCH_INFO_FILE}"
+echo "next=run scripts/launch_health.sh to capture the real optimizer pid"