devalshahamd · Sep 18, 2026
@@ -7,6 +7,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ### Fixed
 
+- **AgentX profiles now sample settled workload with bounded, agentic windows.**
+  Profiling starts 300 seconds after AIPerf enters its measured phase and stops
+  after either 256 engine steps or 100 seconds. AgentX trace analysis now asks
+  TraceLens for a 64-step window with `R=0.1`, the measured concurrency, and an
+  OSL hint of 512 instead of inheriting synthetic-workload splitter defaults.
+
 - **AgentX grading failures no longer fall back to throughput KEEP.** When an
   AgentX session cannot grade on interactivity because either side is missing
   the axis pair, explore, ``_lift_to_current_best``, and

@@ -1000,10 +1000,12 @@ scriptable frameworks (xDiT, custom) keep output-throughput grading.
 | `HYPERLOOM_ALLOW_UNVERIFIED_SUBMISSION` | Unset (fail closed) | Truthy accepts a measurement whose submission verdict is absent or undetermined (`submission_valid=None`). A measurement the scenario explicitly judged invalid (`submission_valid=False`) is always rejected regardless of this flag. Applies to every measurement the run accepts (baseline, explore, kernel, sweep), not only the baseline — an unverified measurement makes every gain derived from it unverifiable. |
 | `INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC` | `7200` | Server-boot budget for the persistent-server phase: how long a launch may spend before the health endpoint answers. Sized for a TB-scale checkpoint — a 1.56 TB MXFP4 MoE reads for ~37 minutes before the first aiter JIT — so it is not AgentX-gated; a synthetic run on the same weights waits the same. A server that never comes up is still bounded by the per-phase and session budgets. |
 
-AgentX profiling starts when AIPerf reports its measured phase. The legacy
-`AGENTX_PROFILE_WARMUP_S` delay is ignored. `AGENTX_PROFILE_WINDOW_S` controls
-the capture window and defaults to 20 seconds; phase waiting is bounded by the
-materialized benchmark timeout. Capture lifecycle status is
+AgentX profiling starts 300 seconds after AIPerf reports its measured phase.
+`AGENTX_PROFILE_START_DELAY_S` overrides that delay, while the legacy
+`AGENTX_PROFILE_WARMUP_S` remains ignored. Capture stops after 256 engine steps
+or `AGENTX_PROFILE_WINDOW_S` seconds, whichever comes first; the window defaults
+to 100 seconds. Phase waiting is bounded by the materialized benchmark timeout.
+Capture lifecycle status is
 written to a per-invocation `capture-status.json`; the adjacent
 `trace-manifest.json` records the selected primary and per-rank traces.
 Benchmark measurement success and trace-capture success are reported

@@ -48,7 +48,8 @@
 #     where NO trace file appears at all -- a failed capture, not a slow one;
 #     default 900, clamped to AGENTX_TRACE_FLUSH_TIMEOUT_S. The first rank file
 #     landed at t+350s on that same GLM-5.3 capture),
-#   AGENTX_KEEP_SERVER, AGENTX_PROFILE_WINDOW_S,
+#   AGENTX_KEEP_SERVER, AGENTX_PROFILE_START_DELAY_S,
+#   AGENTX_PROFILE_WINDOW_S,
 #   AGENTX_PROFILE_WARMUP_S (deprecated and ignored; phase-gated profiling
 #     replaced the fixed warmup delay),
 #   AGENTX_SERVER_SCRIPT (override builtin name), AIPERF_BIN.
@@ -573,8 +574,10 @@ if [ "${PROFILE:-0}" = "1" ]; then
   # framework's --profiler-config/env when PROFILE=1); /start_profile begins
   # recording and /stop_profile flushes the trace to torch_profiler_dir for
   # TraceLens. Only fires under PROFILE=1, so measurement rounds pay no cost.
-  PWIN="${AGENTX_PROFILE_WINDOW_S:-20}"
+  PWIN="${AGENTX_PROFILE_WINDOW_S:-100}"
   _require_uint AGENTX_PROFILE_WINDOW_S "$PWIN"
+  PROFILE_START_DELAY="${AGENTX_PROFILE_START_DELAY_S:-300}"
+  _require_uint AGENTX_PROFILE_START_DELAY_S "$PROFILE_START_DELAY"
   : "${AGENTX_CAPTURE_ID:?AGENTX_CAPTURE_ID required for AgentX profiling}"
   : "${AGENTX_CAPTURE_STATUS_PATH:?AGENTX_CAPTURE_STATUS_PATH required for AgentX profiling}"
   PHASE_WAIT_TIMEOUT="${AGENTX_PHASE_WAIT_TIMEOUT_S:-$(( DATASET_CONFIG_TIMEOUT + WARMGRACE + DURATION ))}"
@@ -740,6 +743,14 @@ if [ "${PROFILE:-0}" = "1" ]; then
       --timeout-seconds "$PHASE_WAIT_TIMEOUT"
   )"; then
     log "AIPerf measured phase started (start_ns=${PHASE_START_NS})"
+    if [ "$PROFILE_START_DELAY" -gt 0 ]; then
+      log "waiting ${PROFILE_START_DELAY}s after measured phase before starting the profiler"
+      sleep "$PROFILE_START_DELAY"
+    fi
+    if ! kill -0 "$APID" 2>/dev/null; then
+      log "WARN AIPerf exited before the profile start delay elapsed; skipping trace capture"
+      _write_profile_capture_status "failed" "profiling_phase_unavailable" "$PHASE_START_NS"
+    else
     # SGLang takes its capture bounds in the /start_profile BODY, not on the
     # serve line. Hyperloom computes them into $PROFILE_EXTRA_BODY, but only
     # InferenceX's own client ever posted it -- a bare POST leaves the capture
@@ -778,18 +789,22 @@ if [ "${PROFILE:-0}" = "1" ]; then
       # Auto-stop validation and any subsequent flush share one completion budget.
       TRACE_FLUSH_START_NS=$(date +%s%N)
       TRACE_FLUSH_DEADLINE_NS=$(( TRACE_FLUSH_START_NS + TRACE_FLUSH_BUDGET * 1000000000 ))
-      # Native num_steps may already have stopped SGLang. Never POST a redundant
-      # stop unless current complete traces cannot prove that auto-stop finished.
+      NATIVE_COMPLETE=0
       if [ "$AUTO_BOUNDED" -eq 1 ] && _trace_complete; then
+        NATIVE_COMPLETE=1
+      fi
+      # Always issue the explicit stop at the client boundary. A native
+      # num_steps completion may have stopped the profiler first.
+      if curl -sf -X POST "http://localhost:${PORT}/stop_profile" >/dev/null 2>&1; then
+        STOP_OK=1
+        log "stop_profile OK"
+      elif [ "$NATIVE_COMPLETE" -eq 1 ]; then
         STOP_OK=1
-        log "profile auto-completed: current GPU traces cover all expected ranks; skipping stop_profile"
+        log "stop_profile was already satisfied by native profile completion"
       else
-        if curl -sf -X POST "http://localhost:${PORT}/stop_profile" >/dev/null 2>&1; then
-          STOP_OK=1
-          log "stop_profile OK"
-        else
-          log "WARN stop_profile failed"
-        fi
+        log "WARN stop_profile failed"
+      fi
+      if [ "$NATIVE_COMPLETE" -ne 1 ]; then
         _wait_for_trace_flush || FLUSH_RC=$?
       fi
       if [ "$STOP_OK" -eq 1 ] && [ "$FLUSH_RC" -eq 0 ]; then
@@ -809,6 +824,7 @@ if [ "${PROFILE:-0}" = "1" ]; then
       log "WARN start_profile failed (trace may be empty)"
       _write_profile_capture_status "failed" "start_profile_failed" "$PHASE_START_NS"
     fi
+    fi
   else
     log "WARN AIPerf did not expose a measured phase; the measurement will finish without trace capture"
     _write_profile_capture_status "failed" "$PHASE_GATE_FAILURE_REASON"

@@ -444,8 +444,11 @@ def _client_only_env(bind, tmp_path):
     bash_env = tmp_path / "bash-env.sh"
     bash_env.write_text(
         f"""kill() {{
+  if [ "${{1:-}}" = "-0" ]; then
+    command kill "$@"
+    return
+  fi
   printf 'kill %s\\n' "$*" >> {lifecycle_marker}
-  [ "${{1:-}}" != "-0" ]
 }}
 """,
         encoding="utf-8",
@@ -836,13 +839,14 @@ def _run_profile(bench, bind, res, tmp_path, **extra_env):
     """PROFILE=1 with the window collapsed, so the branch runs in seconds."""
     capture_dir = res / "agentx-profile" / "test-capture"
     capture_dir.mkdir(parents=True, exist_ok=True)
+    extra_env.setdefault("AGENTX_PROFILE_WINDOW_S", "0")
+    extra_env.setdefault("AGENTX_PROFILE_START_DELAY_S", "0")
     return _run(
         bench,
         bind,
         res,
         tmp_path,
         PROFILE="1",
-        AGENTX_PROFILE_WINDOW_S="0",
         AGENTX_CAPTURE_ID="test-capture",
         AGENTX_CAPTURE_STATUS_PATH=str(capture_dir / "capture-status.json"),
         FAKE_AIPERF_SLEEP="6",
@@ -855,6 +859,41 @@ def _capture_status_path(res: Path) -> Path:
     return res / "agentx-profile" / "test-capture" / "capture-status.json"
 
 
+def test_profile_waits_after_measured_phase_before_starting(tmp_path):
+    bench, bind, res = _sandbox(tmp_path)
+    delay_marker = tmp_path / "profile-delay.txt"
+    bash_env = tmp_path / "profile-delay-env.sh"
+    bash_env.write_text(
+        f"""sleep() {{
+  if [ "${{1:-}}" = "300" ]; then
+    printf '%s\\n' "$1" > {shlex.quote(str(delay_marker))}
+    return
+  fi
+  command sleep "$@"
+}}
+""",
+        encoding="utf-8",
+    )
+    result = _run_profile(
+        bench,
+        bind,
+        res,
+        tmp_path,
+        AGENTX_PROFILE_START_DELAY_S="300",
+        BASH_ENV=str(bash_env),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert delay_marker.read_text(encoding="utf-8").strip() == "300"
+
+
+def test_profile_window_defaults_to_100_seconds(tmp_path):
+    bench, bind, res = _sandbox(tmp_path)
+    result = _run_profile(bench, bind, res, tmp_path, AGENTX_PROFILE_WINDOW_S=None)
+    assert result.returncode == 0, result.stdout + result.stderr
+    capture = json.loads(_capture_status_path(res).read_text())
+    assert capture["requested_window_seconds"] == 100
+
+
 def _fast_trace_poll_env(bind, tmp_path):
     """Advance shell polling time without changing the real phase gate's clock."""
     env = _client_only_env(bind, tmp_path)
@@ -914,6 +953,7 @@ def test_client_only_profiles_measured_phase_without_server_cleanup(tmp_path, st
         SGLANG_TORCH_PROFILER_DIR="",
         VLLM_TORCH_PROFILER_DIR="",
         AGENTX_PROFILE_WINDOW_S="20",
+        AGENTX_PROFILE_START_DELAY_S="0",
         AGENTX_PHASE_WAIT_TIMEOUT_S="10",
         AGENTX_CAPTURE_ID="test-capture",
         AGENTX_CAPTURE_STATUS_PATH=str(_capture_status_path(res)),
@@ -949,7 +989,7 @@ def test_client_only_profiles_measured_phase_without_server_cleanup(tmp_path, st
 @pytest.mark.parametrize(
     "framework,body,case,stop_rc,reason,stop_called",
     [
-        ("sglang", '{"num_steps":8}', "complete", "22", "capture_complete", False),
+        ("sglang", '{"num_steps":8}', "complete", "22", "capture_complete", True),
         ("sglang", '{"num_steps":8}', "stale", "0", "trace_files_missing", True),
         ("sglang", '{"num_steps":8}', "partial_ranks", "0", "trace_flush_timeout", True),
         ("sglang", '{"num_steps":8}', "bad_gzip", "0", "trace_flush_timeout", True),
@@ -958,7 +998,7 @@ def test_client_only_profiles_measured_phase_without_server_cleanup(tmp_path, st
         ("sglang", '{"num_steps":true}', "complete", "22", "stop_profile_failed", True),
     ],
 )
-def test_stop_is_skipped_only_for_proven_current_native_completion(
+def test_stop_is_sent_even_after_proven_current_native_completion(
     tmp_path, framework, body, case, stop_rc, reason, stop_called
 ):
     bench, bind, res = _sandbox(tmp_path, write_pid=False)
@@ -1416,6 +1456,7 @@ def test_a_framework_script_disagreement_is_said_out_loud(tmp_path):
     "knob,value",
     [
         ("AGENTX_PROFILE_WINDOW_S", "20.5"),
+        ("AGENTX_PROFILE_START_DELAY_S", "300.5"),
         ("AGENTX_DURATION", "3600.0"),
     ],
 )

@@ -4675,6 +4675,29 @@ def test_agentx_bypass_requires_single_rank(self, monkeypatch, tmp_path):
         assert "--require-single-rank" in cmd
         assert cmd[cmd.index("--tensor-parallel-size") + 1] == "8"
 
+    def test_agentx_tracelens_uses_agentic_splitter_hints(self, monkeypatch, tmp_path):
+        state, session_dir = self._common(monkeypatch, tmp_path)
+        state.benchmark_mode = "agentx"
+        cmd, _steady = krh._build_trace_analyze_cmd(
+            {"trace_input": "/t/trace"},
+            session_dir=session_dir,
+            state=state,
+            workspace_path="/ws",
+            trace_input="/t/trace",
+            tracelens_root=Path("/tl"),
+            is_bypass=False,
+            scriptable=False,
+            workload={"conc": 32, "osl": 4096, "random_range_ratio": "1.0"},
+            model_name="",
+            framework="sglang",
+            target_platform="",
+            analysis_mode="inference",
+        )
+        assert cmd[cmd.index("--split-num-steps") + 1] == "64"
+        assert cmd[cmd.index("--split-r") + 1] == "0.1"
+        assert cmd[cmd.index("--split-conc") + 1] == "32"
+        assert cmd[cmd.index("--split-osl") + 1] == "512"
+
     def test_steady_state_mode_from_env(self, monkeypatch, tmp_path):
         state, session_dir = self._common(monkeypatch, tmp_path)
         monkeypatch.setenv("INFERENCE_OPTIMIZER_STEADY_STATE_MODE", "median")

@@ -837,66 +837,54 @@ def test_materialize_profile_window_sglang_skill_formula(
     assert body["num_steps"] == 128
 
 
-def test_materialize_profile_agentx_clamp_warns_below_steady_floor(
+def test_materialize_profile_agentx_uses_its_own_capture_length(
     tmp_path,
     monkeypatch,
-    caplog,
 ):
-    """AgentX's tighter capture cap (8) must warn when it undercuts steady_floor."""
+    """AgentX capture length is independent of the synthetic 128-step cap."""
     import yaml
 
     _clear_workload_env(monkeypatch)
     monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
     src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024})
-    with caplog.at_level("WARNING"):
-        out = _materialize_config_with_envs(src, tmp_path)
+    out = _materialize_config_with_envs(src, tmp_path)
     rendered = yaml.safe_load(out.read_text())
     extra = rendered["benchmark"]["envs"]["EXTRA_VLLM_ARGS"]
-    assert "--profiler-config.max_iterations 8" in extra, extra
-    assert any("steady-state floor" in r.message for r in caplog.records)
+    assert "--profiler-config.max_iterations 256" in extra, extra
 
 
-def test_materialize_profile_agentx_clamp_warns_on_explicit_override(
+def test_materialize_profile_agentx_ignores_synthetic_steps_cap(
     tmp_path,
     monkeypatch,
-    caplog,
 ):
-    """An explicit HYPERLOOM_PROFILE_MAX_STEPS_CAP must not be silently overridden."""
+    """HYPERLOOM_PROFILE_MAX_STEPS_CAP only controls synthetic profiles."""
     import yaml
 
     _clear_workload_env(monkeypatch)
     monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
     monkeypatch.setenv("HYPERLOOM_PROFILE_MAX_STEPS_CAP", "64")
     src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024})
-    with caplog.at_level("WARNING"):
-        out = _materialize_config_with_envs(src, tmp_path)
+    out = _materialize_config_with_envs(src, tmp_path)
     rendered = yaml.safe_load(out.read_text())
     extra = rendered["benchmark"]["envs"]["EXTRA_VLLM_ARGS"]
-    assert "--profiler-config.max_iterations 8" in extra, extra
-    assert any("explicit HYPERLOOM_PROFILE_MAX_STEPS_CAP=64" in r.message for r in caplog.records)
+    assert "--profiler-config.max_iterations 256" in extra, extra
 
 
-def test_materialize_profile_max_iters_override_warns_it_undoes_the_agentx_bound(
+def test_materialize_profile_max_iters_overrides_agentx_default(
     tmp_path,
     monkeypatch,
-    caplog,
 ):
-    """Overriding the AgentX capture bound must say so -- 128 warns about nothing else."""
+    """HYPERLOOM_PROFILE_MAX_ITERS remains the explicit AgentX override."""
     import yaml
 
     _clear_workload_env(monkeypatch)
     monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
     monkeypatch.setenv("HYPERLOOM_PROFILE_MAX_ITERS", "128")
     src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024})
-    with caplog.at_level("WARNING"):
-        out = _materialize_config_with_envs(src, tmp_path)
+    out = _materialize_config_with_envs(src, tmp_path)
     rendered = yaml.safe_load(out.read_text())
     extra = rendered["benchmark"]["envs"]["EXTRA_VLLM_ARGS"]
-    # The override is still honored verbatim; this is a visibility fix only.
     assert "--profiler-config.max_iterations 128" in extra, extra
-    assert any(
-        "HYPERLOOM_PROFILE_MAX_ITERS=128 overrides the AgentX capture bound of 8" in r.message for r in caplog.records
-    ), [r.message for r in caplog.records]
 
 
 def test_materialize_profile_max_iters_override_is_quiet_without_agentx(