Pin InferenceX and run AgentX with Magpie-managed serving

Magpie/benchmark_config.yaml.example

@@ -307,6 +307,12 @@ benchmark:
 #   EVAL_MAX_MODEL_LEN  - lm-eval local-completions context length
 #   EVAL_MAX_GEN_TOKS   - lm-eval generation budget
 #
+# Magpie's built-in SGLang/vLLM/Atom scripts supply the upstream local-eval
+# defaults EVAL_ONLY=false, IS_AGENTIC=0, and OPENAI_API_KEY=EMPTY. Explicit
+# benchmark.envs values take precedence. Local runs also preserve existing
+# process-environment values; Docker runs do not inherit a host API key.
+# These defaults do not change AgentX recipes or custom benchmark scripts.
+#
 # vLLM specific:
 #   EXTRA_VLLM_ARGS     - Additional vLLM server arguments
 #   VLLM_ROCM_USE_AITER - Enable AITER for AMD ROCm (0/1)

Magpie/config.yaml

@@ -194,9 +194,10 @@ benchmark:
   # Path to separate image mapping configuration file
   image_config: "./benchmark_images.yaml"
 
-  # Path to InferenceX installation
+  # InferenceX repository root or its inferencex-e2e project directory
   # If not configured or path doesn't exist, InferenceX will be auto-cloned from:
   # https://github.com/SemiAnalysisAI/InferenceX.git
+  # New clones pin commit 408c015be4b22d14c69518643609669405507077; existing checkouts are preserved.
   inferencex_path: ""
 
   # Default timeout for benchmark tasks

Magpie/main.py

@@ -1058,6 +1058,11 @@ def run_benchmark(args, config: Dict[str, Any]) -> int:
             "profiler": {
                 "torch_profiler": {
                     "enabled": args.torch_profiler,
+                    "num_steps": getattr(args, "torch_profiler_steps", 20),
+                    "num_profiles": getattr(args, "torch_profiler_count", 1),
+                    "interval_seconds": getattr(args, "torch_profiler_interval", 200.0),
+                    "start_seconds": getattr(args, "torch_profiler_start_seconds", 0.0),
+                    "detailed_annotations": getattr(args, "torch_profiler_detailed_annotations", False),
                 },
                 "system_profiler": {
                     "enabled": args.system_profiler,
@@ -1081,8 +1086,8 @@ def run_benchmark(args, config: Dict[str, Any]) -> int:
     if run_mode:
         benchmark_cfg["run_mode"] = run_mode
 
-    # AgentX remains an InferenceX-owned workload. This flag only asks Magpie
-    # to select and execute the matching InferenceX AgentX launcher/recipe.
+    # AgentX preserves the InferenceX workload while Magpie owns serving for
+    # the packaged client layout; old checkouts retain their launcher path.
     if getattr(args, "agentx", False):
         benchmark_cfg["agentx"] = {
             "enabled": True,
@@ -1297,14 +1302,34 @@ def create_parser() -> argparse.ArgumentParser:
         "--torch-profiler", action="store_true",
         help="Enable torch profiler"
     )
+    benchmark_parser.add_argument(
+        "--torch-profiler-steps", type=int, default=20,
+        help="Server execution steps to capture for AgentX torch profiling (default: 20)",
+    )
+    benchmark_parser.add_argument(
+        "--torch-profiler-count", type=int, default=1,
+        help="Sequential AgentX captures in one replay (default: 1)",
+    )
+    benchmark_parser.add_argument(
+        "--torch-profiler-interval", type=float, default=200.0,
+        help="Seconds after trace flushing before the next AgentX capture (default: 200; 0 starts immediately)",
+    )
+    benchmark_parser.add_argument(
+        "--torch-profiler-start-seconds", type=float, default=0.0,
+        help="Delay from AgentX measurement start before the first capture (default: 0)",
+    )
+    benchmark_parser.add_argument(
+        "--torch-profiler-detailed-annotations", action="store_true",
+        help="Require enhanced AgentX traces from a TraceLens-capable framework runtime",
+    )
     benchmark_parser.add_argument(
         "--system-profiler", action="store_true",
         help="Enable system profiler (rocprof/ncu)"
     )
     benchmark_parser.add_argument(
         "--agentx",
         action="store_true",
-        help="Run the matching InferenceX AgentX trace-replay launcher",
+        help="Run AgentX trace replay with the matching InferenceX recipe",
     )
     benchmark_parser.add_argument(
         "--agentx-mode",
@@ -1324,7 +1349,7 @@ def create_parser() -> argparse.ArgumentParser:
     benchmark_parser.add_argument(
         "--inferencex-path", type=str,
         default="",
-        help="Path to InferenceX installation (auto-cloned if not specified)"
+        help="InferenceX repository or inferencex-e2e directory (auto-cloned at a pinned commit)"
     )
     benchmark_parser.add_argument(
         "--benchmark-script", type=str,

Magpie/mcp/server.py

@@ -1168,7 +1168,7 @@ async def benchmark(
     concurrency: int = 32,
     input_len: Optional[int] = None,
     output_len: Optional[int] = None,
-    torch_profiler: bool = True,
+    torch_profiler: Optional[bool] = None,
     system_profiler: bool = False,
     tracelens: bool = False,
     tracelens_export_format: str = "csv",
@@ -1196,15 +1196,20 @@ async def benchmark(
     ray_num_nodes: int = 1,
     agentx: bool = False,
     agentx_mode: str = "canonical",
+    torch_profiler_steps: int = 20,
+    torch_profiler_count: int = 1,
+    torch_profiler_interval_seconds: float = 200.0,
+    torch_profiler_start_seconds: float = 0.0,
+    torch_profiler_detailed_annotations: bool = False,
 ) -> str:
     """
     Run a framework-level LLM inference benchmark (vLLM, SGLang, or Atom).
 
     Launches the inference server using InferenceX scripts, runs a benchmark
     client, and collects throughput/latency metrics. Ordinary serving
     benchmarks can optionally collect torch profiler traces, run TraceLens
-    analysis, and perform gap analysis. AgentX currently reports AIPerf and
-    server metrics but does not support Magpie profiler collection.
+    analysis, and perform gap analysis. AgentX can explicitly collect bounded
+    torch profiler traces as a diagnostic run; its metrics cannot be published.
 
     InferenceX is auto-cloned if not present.
 
@@ -1222,7 +1227,16 @@ async def benchmark(
         concurrency: Request concurrency (default: 32)
         input_len: Fixed-sequence input length (default: 1024); ignored by AgentX
         output_len: Fixed-sequence output length (default: 512); ignored by AgentX
-        torch_profiler: Enable PyTorch profiler traces (default: True)
+        torch_profiler: Enable PyTorch profiler traces (default: True for ordinary
+            benchmarks, False for AgentX). Explicit True enables AgentX diagnostics.
+        torch_profiler_steps: Server execution steps to capture for AgentX (default: 20).
+        torch_profiler_count: Sequential profiles in one AgentX replay (default: 1).
+        torch_profiler_interval_seconds: Delay after trace flushing before the next
+            AgentX capture (default: 200). Zero starts the next capture immediately.
+        torch_profiler_start_seconds: Delay from AgentX measurement start before
+            the first capture (default: 0).
+        torch_profiler_detailed_annotations: Require enhanced framework annotations
+            and graph shape traces (default: False); requires a capable runtime.
         system_profiler: Enable system profiler - rocprof (AMD) or ncu (NVIDIA) (default: False)
         tracelens: Enable TraceLens trace analysis on host after benchmark (default: False)
         tracelens_export_format: TraceLens export format - "csv" or "excel" (default: "csv")
@@ -1238,7 +1252,7 @@ async def benchmark(
             Default: ["gpu_user_annotation"]
         docker_image: Override automatic Docker image selection (optional)
         gpu_arch: Force GPU architecture, e.g. "gfx942" (auto-detected if omitted)
-        inferencex_path: Path to InferenceX installation (auto-cloned if empty)
+        inferencex_path: InferenceX repository or inferencex-e2e project directory (auto-cloned at a pinned commit if empty)
         hf_cache_path: HuggingFace cache directory (default: ~/.cache/huggingface)
         benchmark_script: Override benchmark script name (e.g., "dsr1_fp8_mi300x.sh")
         runner_type: Hardware runner type (e.g., "mi300x", "h100") - auto-detected if omitted
@@ -1255,7 +1269,7 @@ async def benchmark(
         ray_multi_node: Whether the benchmark needs multiple nodes (default: False)
         ray_total_num_gpus: Total GPUs across nodes for multi-node (default: 8)
         ray_num_nodes: Number of nodes for multi-node (default: 1)
-        agentx: Run the matching InferenceX AgentX launcher and recipe
+        agentx: Run trace replay with the matching InferenceX AgentX recipe
         agentx_mode: "canonical" (publishable) or "fast" (validation only)
 
     Returns:
@@ -1291,12 +1305,18 @@ async def benchmark(
             envs.update(extra_envs)
 
         profiler_cfg = {
-            # AgentX owns its profiling loop. The MCP's historical torch
-            # profiler default is true, so disable it automatically here.
-            "torch_profiler": {"enabled": torch_profiler and not agentx},
+            "torch_profiler": {
+                "enabled": (not agentx) if torch_profiler is None else torch_profiler,
+                "num_steps": torch_profiler_steps,
+                "num_profiles": torch_profiler_count,
+                "interval_seconds": torch_profiler_interval_seconds,
+                "start_seconds": torch_profiler_start_seconds,
+                "detailed_annotations": torch_profiler_detailed_annotations,
+            },
             "system_profiler": {"enabled": system_profiler},
             "tracelens": {
                 "enabled": tracelens,
+                **({"analysis_mode": "pytorch"} if agentx else {}),
                 "export_format": tracelens_export_format,
                 "perf_report_enabled": True,
                 "multi_rank_report_enabled": tp > 1,