fix(hyperloom): classify engine capacity start failures

src/hyperloom/common/failure_signature.py

@@ -294,6 +294,9 @@ def _grp(match: re.Match[str]) -> str:
             re.compile(r"[Cc][Uu][Dd][Aa]\s+out\s+of\s+memory"),
             re.compile(r"no\s+GPU\s+memory\s+(?:for|left\s+for)\s+the\s+KV\s+[Cc]ache", re.IGNORECASE),
             re.compile(r"[Cc]annot\s+allocate\s+(?:memory|cuda|hip)"),
+            re.compile(r"max_num_seqs \(\d+\) exceeds available \w+ cache blocks"),
+            re.compile(r"KV cache is needed, which is larger than the available KV cache memory"),
+            re.compile(r"No available memory for the cache blocks"),
             re.compile(r"[Oo]ut[Oo]f[Mm]emory[Ee]rror"),
             re.compile(r"requires?\s+(?:at\s+least\s+)?(\d+)\s+GPU[s]?"),
             re.compile(r"[Tt]ensor\s+[Pp]arallel\s+[Ss]ize\s+.*?(\d+).*?GPU"),

src/hyperloom/common/tests/test_failure_signature.py

@@ -465,6 +465,20 @@ def test_resource_constraint_tp() -> None:
     assert sig.kind == RESOURCE_CONSTRAINT
 
 
+def test_engine_capacity_is_a_resource_constraint_not_a_kernel_or_arch_gap() -> None:
+    """vLLM start-time capacity errors say the card holds less than the launch asked for."""
+    for text in (
+        "ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (237). Each decode sequence "
+        "requires one Mamba cache block, so CUDA graph capture cannot proceed.",
+        "ValueError: To serve at least one request with the model's max seq len (262144), (24.5 GiB KV cache is "
+        "needed, which is larger than the available KV cache memory (9.1 GiB).",
+        "ValueError: No available memory for the cache blocks. Try increasing `gpu_memory_utilization`",
+    ):
+        sig = classify_failure(text)
+        assert sig.kind == RESOURCE_CONSTRAINT, text
+        assert sig.kind not in (HIP_KERNEL_MISSING, MISSING_MODEL_ARCH), text
+
+
 def test_resource_constraint_no_kv_cache() -> None:
     sig = classify_failure("No GPU memory left for the KV cache. Please try enabling quantization")
     assert sig.kind == RESOURCE_CONSTRAINT

src/hyperloom/inference_optimizer/tests/test_baseline_failure_summary.py

@@ -98,6 +98,25 @@ def test_classify_root_cause_type():
     assert _classify_root_cause_type("unknown", "") == "unknown"
 
 
+def test_classify_root_cause_type_engine_capacity():
+    cause = "ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (237)."
+    assert _classify_root_cause_type("engine_capacity", cause) == "engine_capacity"
+    # The class is recovered from the text when only a generic class was recorded.
+    assert _classify_root_cause_type("server_init_dead", f"EngineCore failed to start\n{cause}") == "engine_capacity"
+
+
+def test_failure_summary_headline_is_the_capacity_line():
+    cause = "ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (237). Lower max_num_seqs."
+    state = _failed_state(
+        error_class="engine_capacity",
+        error=f"(EngineCore pid=7) EngineCore failed to start.\n{cause}",
+    )
+    fs = _build_failure_summary(state)
+    assert fs is not None
+    assert fs["root_cause_type"] == "engine_capacity"
+    assert fs["root_cause"] == cause
+
+
 def test_failure_summary_from_server_init_dead_attempt():
     state = _failed_state(error_class="server_init_dead", error=_SERVER_LOG_OOM)
     fs = _build_failure_summary(state)

src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py

@@ -1721,6 +1721,64 @@ def fake_run(cmd, *args, **kwargs):
     assert "Engine core initialization failed" in result["error"]
 
 
+_MAMBA_CAPACITY_ERROR = (
+    "(EngineCore pid=7) ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (237). "
+    "Each decode sequence requires one Mamba cache block, so CUDA graph capture cannot proceed."
+)
+
+
+def _engine_capacity_server_log() -> str:
+    """vLLM writes the start-failure marker first and the traceback, ending in the cause, after it."""
+    frames = "".join(f"(EngineCore pid=7)     frame {i}\n" for i in range(55))
+    return (
+        "(EngineCore pid=7) ERROR core.py:1349] EngineCore failed to start.\n"
+        "(EngineCore pid=7) Traceback (most recent call last):\n"
+        f"{frames}{_MAMBA_CAPACITY_ERROR}\n"
+        "(APIServer pid=1) RuntimeError: Engine core initialization failed. Failed core proc(s): {}\n"
+    )
+
+
+@pytest.mark.parametrize("with_workspace", [False, True])
+def test_baseline_classifies_engine_capacity_separately_from_server_init_dead(
+    tmp_path,
+    with_workspace,
+):
+    """A vLLM capacity error keeps its own class and its cause line, with or without a benchmark_* workspace."""
+    base = tmp_path / "base.yaml"
+    _write_yaml(base, framework="vllm")
+    output_dir = tmp_path / "ws"
+
+    def fake_run(cmd, *args, **kwargs):
+        out_idx = cmd.index("--output-dir")
+        slot = Path(cmd[out_idx + 1])
+        slot.mkdir(parents=True, exist_ok=True)
+        if with_workspace:
+            (slot / "benchmark_vllm_20260602_010101").mkdir(parents=True)
+        (slot / "server.log").write_text(_engine_capacity_server_log(), encoding="utf-8")
+        return subprocess.CompletedProcess(cmd, 1, "", "magpie wrapper noise")
+
+    executor = _executor(base, tmp_path)
+    ctx = _make_ctx(
+        {
+            "baseline_double_run": False,
+            "output_dir": str(output_dir),
+            "timeout_sec": 10,
+            "gpu_type": "mi300x",
+        }
+    )
+
+    with patch(
+        "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill",
+        side_effect=fake_run,
+    ):
+        result = _run(executor(ctx))
+
+    assert result["status"] == "failed"
+    assert result["error_class"] == "engine_capacity", result
+    assert "max_num_seqs (256) exceeds available Mamba cache blocks (237)" in result["error"]
+    assert result["capacity"] == {"kind": "mamba_cache_blocks", "requested": 256, "available": 237}
+
+
 def test_baseline_invalid_measurement_with_server_death_marker_is_dead(
     tmp_path,
     monkeypatch,

src/hyperloom/inference_optimizer/tests/test_failure_evidence.py

@@ -51,6 +51,9 @@ def test_environment_evidence_overrides_generic_failure_class(self):
     def test_ambiguous_failure_remains_unknown(self):
         assert classify_failure_attribution(error_class="server_init_dead") == FAILURE_ATTRIBUTION_UNKNOWN
 
+    def test_engine_capacity_is_not_attributed_to_the_candidate_or_the_host(self):
+        assert classify_failure_attribution(error_class="engine_capacity") == FAILURE_ATTRIBUTION_UNKNOWN
+
     def test_valid_explicit_attribution_wins(self):
         assert (
             classify_failure_attribution(

src/hyperloom/inference_optimizer/tests/test_grid_runner_kill_and_branches_unit.py

@@ -434,3 +434,30 @@ def _dead(magpie_python, config_path, output_dir, **_k):
     assert "mla_gluon" in (r.error or ""), "excerpt should mention mla_gluon"
     assert r.server_log_path is not None
     assert r.server_log_path.endswith("server.log")
+
+
+@pytest.mark.asyncio
+async def test_server_dead_with_a_capacity_cause_is_engine_capacity(tmp_path, monkeypatch):
+    """A watchdog-reaped variant whose server.log ends in the capacity error is not a generic init death."""
+    base = tmp_path / "base.yaml"
+    _write_base_yaml(base)
+    cause = "ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (237)."
+
+    def _dead(magpie_python, config_path, output_dir, **_k):
+        slog = Path(output_dir) / "server.log"
+        slog.parent.mkdir(parents=True, exist_ok=True)
+        frames = "".join(f"    frame {i}\n" for i in range(55))
+        slog.write_text(f"EngineCore failed to start.\nTraceback (most recent call last):\n{frames}{cause}\n")
+        return gr.SERVER_DEAD_RETURNCODE, "", ""
+
+    monkeypatch.setattr(gr, "_run_magpie", _dead)
+    results = await run_grid(
+        base_yaml_path=base,
+        base_extra_args="",
+        grid=[GridVariant("fp8_kv")],
+        output_root=tmp_path / "out",
+    )
+    r = results[0]
+    assert r.status == "failed"
+    assert r.error_class == "engine_capacity"
+    assert cause in (r.error or "")

src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py

@@ -27,6 +27,7 @@
     _scan_logs_increment,
     _scan_server_log_increment,
     _server_log_shows_death,
+    engine_capacity_cause,
     kill_my_spawned_server,
     new_session_kwargs,
     run_with_session_kill,
@@ -1059,6 +1060,73 @@ def test_server_log_death_excerpt_surfaces_config_validation_arch_miss(tmp_path)
     assert sig.offending_symbol == "deepseek_v4"
 
 
+_MAMBA_CAPACITY_LINE = (
+    "(EngineCore pid=7) ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (237). "
+    "Each decode sequence requires one Mamba cache block, so CUDA graph capture cannot proceed. "
+    "Please lower max_num_seqs to at most 237 or increase gpu_memory_utilization."
+)
+_KV_MEMORY_LINE = (
+    "ValueError: To serve at least one request with the model's max seq len (262144), (24.5 GiB KV cache is "
+    "needed, which is larger than the available KV cache memory (9.1 GiB). Try increasing "
+    "`gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine."
+)
+_KV_EMPTY_LINE = (
+    "ValueError: No available memory for the cache blocks. Try increasing `gpu_memory_utilization` "
+    "when initializing the engine."
+)
+
+
+def _engine_start_failure_log(cause: str) -> str:
+    """vLLM logs the marker first and the traceback after it, so the cause is its last line."""
+    frames = "".join(f"(EngineCore pid=7)     frame {i}\n" for i in range(55))
+    return (
+        "(EngineCore pid=7) ERROR core.py:1349] EngineCore failed to start.\n"
+        "(EngineCore pid=7) Traceback (most recent call last):\n"
+        f"{frames}{cause}\n"
+        "(APIServer pid=1) RuntimeError: Engine core initialization failed. "
+        "See root cause above. Failed core proc(s): {}\n"
+    )
+
+
+def test_server_log_death_excerpt_keeps_the_exception_below_the_marker(tmp_path):
+    """``EngineCore failed to start`` precedes its traceback, so the cause sits tens of lines below it."""
+    log_path = tmp_path / "server.log"
+    log_path.write_text(_engine_start_failure_log(_MAMBA_CAPACITY_LINE))
+    excerpt = server_log_death_excerpt(str(log_path))
+    assert excerpt is not None
+    assert "EngineCore failed to start" in excerpt
+    assert "max_num_seqs (256) exceeds available Mamba cache blocks (237)" in excerpt
+
+
+def test_engine_capacity_cause_parses_the_numbers():
+    assert engine_capacity_cause(_MAMBA_CAPACITY_LINE) == {
+        "kind": "mamba_cache_blocks",
+        "requested": 256,
+        "available": 237,
+    }
+    assert engine_capacity_cause(_KV_MEMORY_LINE) == {
+        "kind": "kv_cache_memory",
+        "max_model_len": 262144,
+        "needed_gib": 24.5,
+        "available_gib": 9.1,
+    }
+    assert engine_capacity_cause(_KV_EMPTY_LINE) == {"kind": "kv_cache_empty"}
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "",
+        "RuntimeError: HIP out of memory. Tried to allocate 10.54 GiB",
+        "hipErrorNoBinaryForGpu: no kernel image is available for execution on the device",
+        "Transformers does not recognize this architecture",
+        "INFO max_num_seqs=256 gpu_memory_utilization=0.9",
+    ],
+)
+def test_engine_capacity_cause_ignores_other_failures(text):
+    assert engine_capacity_cause(text) is None
+
+
 def test_fatal_text_does_not_kill_a_running_child(tmp_path):
     script = (
         "import pathlib,sys,time; pathlib.Path(sys.argv[1]).write_text('EngineCore failed to start'); time.sleep(.8)"

src/hyperloom/orchestrator/actions/executors/_grid_runner.py

@@ -53,6 +53,7 @@
     ORCHESTRATOR_CANCELLED_RETURNCODE,
     SERVER_DEAD_RETURNCODE,
     SESSION_TIME_EXHAUSTED_RETURNCODE,
+    engine_capacity_cause,
     resolve_benchmark_timeouts,
     run_with_session_kill,
     server_log_death_excerpt,
@@ -1629,10 +1630,11 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant
                 death_excerpt = server_log_death_excerpt(str(server_log)) or (
                     "server engine/worker init failed; parent process hung and was reaped by the liveness watchdog"
                 )
+                death_class = "engine_capacity" if engine_capacity_cause(death_excerpt) else "server_init_dead"
                 _write_variant_abort_marker(
                     slot,
                     variant_name=variant.name,
-                    error_class="server_init_dead",
+                    error_class=death_class,
                     error_summary=death_excerpt,
                     extra_args=variant.extra_server_args,
                 )
@@ -1645,7 +1647,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant
                         returncode=rc,
                         runtime_sec=variant_runtime_sec,
                         error=death_excerpt,
-                        error_class="server_init_dead",
+                        error_class=death_class,
                         server_log_path=_existing_log_path(server_log),
                         note=variant.note,
                         nonfatal_warnings=[f"harvested_leaked_artifact:{src}" for src, _ in sd_harvested],

src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py

@@ -203,6 +203,37 @@ def kill_my_spawned_server(
 #: Lines of preceding context a legacy bootstrap marker keeps.
 _MARKER_CONTEXT_LINES: int = 2
 
+#: vLLM start-time capacity errors: the card holds less than the launch asked for. Each entry names its kind and
+#: captures the numbers the message carries.
+_ENGINE_CAPACITY_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    (
+        "mamba_cache_blocks",
+        re.compile(r"max_num_seqs \((?P<requested>\d+)\) exceeds available Mamba cache blocks \((?P<available>\d+)\)"),
+    ),
+    (
+        "kv_cache_memory",
+        re.compile(
+            r"max seq len \((?P<max_model_len>\d+)\), \((?P<needed_gib>[\d.]+) GiB KV cache is needed, "
+            r"which is larger than the available KV cache memory \((?P<available_gib>[\d.]+) GiB\)"
+        ),
+    ),
+    ("kv_cache_empty", re.compile(r"No available memory for the cache blocks")),
+)
+
+
+def engine_capacity_cause(text: str) -> dict[str, Any] | None:
+    """Return the parsed vLLM engine-capacity error in ``text``, else ``None``.
+
+    The dict names the ``kind`` and the numbers the message carries; a capacity failure is the host lacking room for
+    the launch settings, so it never reads as a kernel or architecture gap.
+    """
+    for kind, pattern in _ENGINE_CAPACITY_PATTERNS:
+        match = pattern.search(text)
+        if match:
+            numbers = {k: float(v) if "." in v else int(v) for k, v in match.groupdict().items()}
+            return {"kind": kind, **numbers}
+    return None
+
 
 # Sentinel ``returncode`` when the detokenizer-stall watchdog reaps a child that came up healthy but then produced no
 # generation progress (hung engine / detokenizer wedge).
@@ -500,9 +531,11 @@ def _first_marker_excerpt(path: str, *, max_chars: int) -> str | None:
                 # of ``AsyncEngineDeadError`` -- and a legacy line must keep the
                 # legacy extraction, whose cause sits above the marker.
                 if any(marker in line for marker in _SERVER_DEAD_MARKERS):
-                    after = _read_lines(fh, _MARKER_CONTEXT_LINES)
-                    excerpt = "\n".join([*before, line, *after]).strip()
-                    return excerpt[-max_chars:] or None
+                    # ``EngineCore failed to start`` is logged ahead of its traceback, so the cause can sit below.
+                    window = _read_lines(fh, _MARKER_WINDOW_LINES)
+                    head = "\n".join([*before, line, *window[:_MARKER_CONTEXT_LINES]]).strip()[-max_chars:]
+                    causes = _exception_lines(window[_MARKER_CONTEXT_LINES:], room=max_chars - len(head) - 1)
+                    return "\n".join([head, *causes]) or None
                 if any(marker in line for marker in _FATAL_ENGINE_MARKERS):
                     return _cause_first_excerpt(
                         line.strip(), _read_lines(fh, _MARKER_WINDOW_LINES - 1), max_chars=max_chars
@@ -524,6 +557,21 @@ def _read_lines(fh: Any, count: int) -> list[str]:
     return out
 
 
+def _exception_lines(window: list[str], *, room: int) -> list[str]:
+    """Return the lines of ``window`` that name an exception, nearest first, within ``room`` characters."""
+    causes: list[str] = []
+    used = 0
+    for line in window:
+        if not _EXCEPTION_LINE.search(line):
+            continue
+        text = line.strip()
+        if used + len(text) + 1 > room:
+            break
+        causes.append(text)
+        used += len(text) + 1
+    return causes
+
+
 def _cause_first_excerpt(head: str, window: list[str], *, max_chars: int) -> str | None:
     """Return the marker line followed by the exceptions it names, nearest first.
 
@@ -537,16 +585,7 @@ def _cause_first_excerpt(head: str, window: list[str], *, max_chars: int) -> str
     room = max_chars - len(head) - 1
     if room <= 0:
         return head[:max_chars] or None
-    causes: list[str] = []
-    used = 0
-    for line in window:
-        if not _EXCEPTION_LINE.search(line):
-            continue
-        text = line.strip()
-        if used + len(text) + 1 > room:
-            break
-        causes.append(text)
-        used += len(text) + 1
+    causes = _exception_lines(window, room=room)
     if causes:
         return head + "\n" + "\n".join(causes)
     rest = "\n".join(window).strip()