fix(agentic): require GB200 validated checkpoints

Signed-off-by: Cam Quilici <cjquilici@gmail.com>

Author: cquil11

.github/workflows/benchmark-multinode-tmpl.yml

@@ -247,12 +247,48 @@ jobs:
             fi
             # Existence is not enough: process_agentic_result.py writes the
             # aggregate even when aiperf recorded zero valid requests. Require
-            # successful requests in every concurrency result.
+            # successful requests and the success marker staged only after
+            # replay/result validation in every concurrency result.
             for result_file in "${agentic_results[@]}"; do
               echo "Found agentic result file: $result_file"
-              ok=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(int(bool(d.get('num_requests_successful'))))" "$result_file" 2>/dev/null || echo 0)
+              conc=$(sed -n 's/.*_conc\([0-9][0-9]*\)\.json$/\1/p' <<< "$result_file")
+              marker_file="agentic_checkpoints/${RESULT_FILENAME}_conc${conc}.success.json"
+              checkpoint_file="agentic_checkpoints/${RESULT_FILENAME}_conc${conc}.json"
+              if [ -z "$conc" ] || [ ! -f "$marker_file" ] || [ ! -f "$checkpoint_file" ]; then
+                echo "Run failed: $result_file is missing validated AgentX checkpoint marker $marker_file." >&2
+                exit 1
+              fi
+              ok=$(python3 - "$result_file" "$checkpoint_file" "$marker_file" "$RESULT_FILENAME" "$conc" <<'PY' 2>/dev/null || echo 0
+import json
+import sys
+
+result_file, checkpoint_file, marker_file, base_result_filename, concurrency_raw = sys.argv[1:]
+concurrency = int(concurrency_raw)
+with open(result_file) as handle:
+    result = json.load(handle)
+with open(checkpoint_file) as handle:
+    checkpoint = json.load(handle)
+with open(marker_file) as handle:
+    marker = json.load(handle)
+expected_marker = {
+    "schema_version": 1,
+    "base_result_filename": base_result_filename,
+    "result_filename": f"{base_result_filename}_conc{concurrency}.json",
+    "concurrency": concurrency,
+}
+print(
+    int(
+        marker == expected_marker
+        and int(result.get("conc", -1)) == concurrency
+        and int(checkpoint.get("conc", -1)) == concurrency
+        and int(result.get("num_requests_successful", 0)) > 0
+        and int(checkpoint.get("num_requests_successful", 0)) > 0
+    )
+)
+PY
+)
               if [ "$ok" != "1" ]; then
-                echo "Run failed: $result_file has zero successful requests." >&2
+                echo "Run failed: $result_file does not match a validated AgentX checkpoint." >&2
                 exit 1
               fi
             done

experimental/dsv4_agentx_gb200_autoresearch/experiment-ledger.jsonl

@@ -3,3 +3,4 @@
 {"timestamp":"2026-06-26T15:29:00Z","kind":"runtime_sync","inferencex_sha":"ba2049d8308a3c605c8303628d68e7e963198aae","aiperf_sha":"24be4b4b0eff13c0a17692ef2eed1e938c94a6e8","included_commit":"999adab351f127837b9e17659ba5c6d9fd791156","rationale":"Canonical combined AIPerf cancellation-drain and pending-flush fixes plus exact benchmark exit-code propagation."}
 {"timestamp":"2026-07-01T06:03:58Z","kind":"search_space_update","inferencex_base_sha":"3d636030a11ea02d71d2e9a49fd039a5229e9169","aiperf_sha":"0595c16aa7e7a7d853d01a932dcd06ced3d8b94d","removed_configs":["dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8-autoresearch"],"added_configs":["dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-tp8-lowlat-autoresearch","dsv4-fp4-gb200-dynamo-vllm-agentic-1p2d-dep8-tp8-lowlat-autoresearch","dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8-bt16k-autoresearch"],"rationale":"User requested removing worthless TEP configs, adding low-concurrency DEP8 prefill plus TP8 decode low-latency candidates, and testing larger 16k prefill batches to push high-throughput frontier without the observed c192-to-c224 TTFT cliff.","local_validation":"generate_sweep_configs full-sweep produced 12 agentic GB200 configs, all duration=3600, no max-model-len, no TEP/3p2d strings, no duplicate exp-name values."}
 {"timestamp":"2026-07-01T08:36:00Z","kind":"failure_diagnosis","workflow_run_id":28500373166,"workflow_attempt":1,"github_job_id":84476045718,"slurm_job_id":20417,"config":"dsv4_p2x8_d1x8_conc176x192","point":"conc176","inferencex_sha":"9a6feea4273b3d5dbaca65f49d44c03f9bc2039d","aiperf_sha":"0595c16aa7e7a7d853d01a932dcd06ced3d8b94d","observed":"AIPerf aborted during accelerated warmup before profiling with replay_rc=1, Benchmark aborted: warmup_failure, no valid profile export, and mass ServerDisconnectedError after profile cancel.","root_cause":"Server-side prefill EngineCore died; frontend reported EngineDeadError and prefill logs showed crash in fused_moe_trtllm_sm100 under the experimental 16384 max-num-batched-tokens recipe.","decision":"Cap the experimental high-throughput recipe back to 8192 max-num-batched-tokens while preserving the materialized CONFIG_FILE path so the failed matrix job can be rerun on the same workflow."}
+{"timestamp":"2026-07-01T15:56:00Z","kind":"false_green_diagnosis","workflow_run_id":28500373166,"workflow_attempt":1,"github_job_id":84476045810,"slurm_job_id":20423,"config":"dsv4_p2x8_d1x8_conc144x160","point":"conc160","inferencex_sha":"c53253c5434fdb70f05f61a4fd2fb1ac32e8ec1f","aiperf_sha":"0595c16aa7e7a7d853d01a932dcd06ced3d8b94d","observed":"GHA job concluded success even though conc160 AIPerf emitted ProfileAborted(failed_request_threshold), profiling lasted only 25.35s, profile_export_aiperf.json was absent, and checkpoint artifact contained only the conc144 success marker. The bmk artifact nevertheless uploaded a partial conc160 aggregate.","root_cause":"GB200 host launcher treated presence of per-concurrency JSON files with num_requests_successful>0 as sufficient, cancelled server workers, and never required the AgentX success markers that are written only after AIPerf replay/result validation succeeds.","decision":"Require validated per-concurrency AgentX checkpoint JSON plus .success.json marker before early Slurm cancellation and before GB200 launcher success; mirror the same marker check in benchmark-multinode-tmpl for future runs."}

runners/launch_gb200-nv.sh

@@ -513,19 +513,49 @@ agentic_results_ready() {
         return 1
     fi
 
-    local expected_count
-    expected_count=$(wc -w <<<"$CONC_LIST" | tr -d ' ')
-    shopt -s nullglob
-    local agentic_results=("${RESULT_FILENAME}"_conc*.json)
-    shopt -u nullglob
-    if [[ "${#agentic_results[@]}" -ne "$expected_count" ]]; then
-        return 1
-    fi
-
+    local checkpoint_dir="${AGENTIC_CHECKPOINT_DIR:-${GITHUB_WORKSPACE:-$PWD}/agentic_checkpoints}"
+    local concurrency
     local result_file
-    local ok
-    for result_file in "${agentic_results[@]}"; do
-        ok=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(int(bool(d.get('num_requests_successful'))))" "$result_file" 2>/dev/null || echo 0)
+    local checkpoint_result
+    local marker_file
+
+    for concurrency in $CONC_LIST; do
+        result_file="${RESULT_FILENAME}_conc${concurrency}.json"
+        checkpoint_result="${checkpoint_dir}/${RESULT_FILENAME}_conc${concurrency}.json"
+        marker_file="${checkpoint_dir}/${RESULT_FILENAME}_conc${concurrency}.success.json"
+        if [[ ! -f "$result_file" || ! -f "$checkpoint_result" || ! -f "$marker_file" ]]; then
+            return 1
+        fi
+        local ok
+        ok=$(python3 - "$result_file" "$checkpoint_result" "$marker_file" "$RESULT_FILENAME" "$concurrency" <<'PY' 2>/dev/null || echo 0
+import json
+import sys
+
+result_file, checkpoint_result, marker_file, base_result_filename, concurrency_raw = sys.argv[1:]
+concurrency = int(concurrency_raw)
+with open(result_file) as handle:
+    result = json.load(handle)
+with open(checkpoint_result) as handle:
+    checkpoint = json.load(handle)
+with open(marker_file) as handle:
+    marker = json.load(handle)
+expected_marker = {
+    "schema_version": 1,
+    "base_result_filename": base_result_filename,
+    "result_filename": f"{base_result_filename}_conc{concurrency}.json",
+    "concurrency": concurrency,
+}
+print(
+    int(
+        marker == expected_marker
+        and int(result.get("conc", -1)) == concurrency
+        and int(checkpoint.get("conc", -1)) == concurrency
+        and int(result.get("num_requests_successful", 0)) > 0
+        and int(checkpoint.get("num_requests_successful", 0)) > 0
+    )
+)
+PY
+)
         if [[ "$ok" != "1" ]]; then
             return 1
         fi
@@ -593,6 +623,13 @@ wait $POLL_PID
 
 set -x
 
+if [[ "${BENCHMARK_TYPE:-}" == "agentic-coding" ]]; then
+    if ! agentic_results_ready; then
+        echo "ERROR: AgentX run finished without validated success markers for every concurrency in CONC_LIST=$CONC_LIST" >&2
+        exit 1
+    fi
+fi
+
 echo "Job $JOB_ID completed!"
 echo "Collecting results..."