@@ -1,24 +1,23 @@
 [server]
 python = "/path/to/model-env/bin/python"
-module = "executorch.examples.models.muse_glimmer.serving.serve"
-worker_bin = "/path/to/executorch/cmake-out/examples/models/muse-glimmer/muse_glimmer_worker"
-model_path = "/path/to/muse-glimmer-k-quant-17G-128K-text-dflash-metal.pte"
+module = "executorch.examples.models.muse_glimmer.serving.serve_batching"
+worker_bin = "/path/to/executorch/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker"
+# Requires an MLX solo export with --use-offgraph-kv-cache.
+model_path = "/path/to/muse-glimmer-offgraph/model.pte"
 # Keep tokenizer.json, tokenizer_config.json, and chat_template.jinja together.
 tokenizer_path = "/path/to/tokenizer/tokenizer.json"
 hf_tokenizer = "/path/to/tokenizer"
 model_id = "muse-glimmer-30b"
 max_context = 131072
-max_sessions = 2 # One scratch session and one named session for cross-turn KV reuse.
 tool_parser = "atem"
-artifact_mode = "dflash"
 # host = "0.0.0.0" # Exposes the server on all network interfaces.
 # port = 8000
 
 [terminal_bench]
-tasks = ["fix-git"]
+tasks = ["fix-git", "git-multibranch", "log-summary-date-ranges", "db-wal-recovery"]
 max_output_tokens = 4096
 temperature = 0.8
 step_limit = 100
 attempts = 1
-session_affinity = "glimmer-terminal-bench"
+concurrency = 4
 # agent_host = "host.lima.internal" # Override automatic Docker host detection.
@@ -44,6 +44,17 @@ Configuration details:
 - Optional `session_affinity` sends an `x-session-affinity` header for KV reuse.
   It requires worker support for named sessions; sequential trials share the ID.
 
+For concurrent tasks, set `[terminal_bench].concurrency` (default `1`), remove
+`session_affinity`, and select a batching server and compatible export. Server
+limits use the launcher's defaults. Keep task concurrency within the server's
+session and request capacity; configure `[server]` options as needed.
+Each trial/attempt uses Harbor's trial UUID as its KV-session ID. Cleanup runs in
+Harbor's END hook before the trial slot is released, retrying transient failures
+for up to two minutes. Persistent cleanup errors stop the run to avoid reusing
+unreleased capacity. Harbor retains the same mini-SWE-agent loop and per-trial
+verification.
+The run also checks that evaluation session IDs match the Harbor trial UUIDs.
+
 `--dry-run` writes resolved commands to `run.json`. Use `--task NAME` to override
 the task list and `--output DIR` to select a new results directory. Each run owns
 its server process.
@@ -60,3 +71,5 @@ its server process.
 
 Model revisions, export/build commands, and dependency versions are not captured;
 retain them alongside results.
+With concurrency, `metrics.json` sums per-request timings, which overlap;
+these sums are not elapsed engine time or aggregate throughput.
@@ -62,6 +62,7 @@ def load_config(path):
     defaults = {
         "tasks": ["fix-git"],
         "attempts": 1,
+        "concurrency": 1,
         "step_limit": 100,
         "max_output_tokens": 512,
         "temperature": 0.0,
@@ -83,6 +84,13 @@ def load_config(path):
     affinity = options["session_affinity"]
     if affinity is not None and (not isinstance(affinity, str) or not affinity.strip()):
         raise ValueError("[terminal_bench].session_affinity must be a nonempty string")
+    concurrency = options["concurrency"]
+    if type(concurrency) is not int or concurrency < 1:
+        raise ValueError("[terminal_bench].concurrency must be a positive integer")
+    if concurrency > 1 and affinity is not None:
+        raise ValueError(
+            "Remove [terminal_bench].session_affinity for concurrent trials"
+        )
     return config
 
 
@@ -140,7 +148,7 @@ def commands(config, output):
         "--ae",
         f"no_proxy={agent_host}",
         "--n-concurrent",
-        "1",
+        str(options["concurrency"]),
         "--n-attempts",
         str(options["attempts"]),
         "--max-retries",
@@ -150,6 +158,18 @@ def commands(config, output):
         "--job-name",
         "harbor",
     ]
+    if options["concurrency"] > 1:
+        module = "executorch.examples.llm_server.evals.terminal_bench.sessions"
+        harbor[harbor.index("--agent") + 1] = f"{module}:SessionMiniSweAgent"
+        host = "127.0.0.1" if server["host"] == "0.0.0.0" else server["host"]
+        harbor.extend(
+            [
+                "--plugin",
+                f"{module}:SessionCleanup",
+                "--plugin-kwarg",
+                f"base_url=http://{host}:{server['port']}/v1",
+            ]
+        )
     for task in options["tasks"]:
         harbor.extend(["--include-task-name", task])
     probe = [
@@ -246,6 +266,7 @@ def metrics(log, context, reserve):
         )
     }
     result["requests"] = len(turns)
+    result["session_ids"] = sorted({turn["session_id"] for turn in turns})
     result["peak_prompt_tokens"] = max(
         (int(turn["prompt_tokens"]) for turn in turns), default=0
     )
@@ -342,6 +363,7 @@ def run(config, output, serve, harbor, probe):
             print("Generation preflight passed.", flush=True)
             job = subprocess.Popen(
                 harbor,
+                env=env,
                 stdout=harbor_log,
                 stderr=subprocess.STDOUT,
                 start_new_session=True,
@@ -388,6 +410,19 @@ def run(config, output, serve, harbor, probe):
         raise RuntimeError(
             "Missing server metrics or context limit exceeded; see metrics.json"
         )
+    if config["terminal_bench"]["concurrency"] > 1:
+        trial_ids = {
+            str(json.loads(path.read_text())["id"])
+            for path in (output / "harbor").glob("*/result.json")
+        }
+        session_ids = set(timing["session_ids"])
+        if session_ids != trial_ids:
+            raise RuntimeError(
+                "Concurrent session IDs do not match Harbor trial UUIDs: "
+                f"unexpected={sorted(session_ids - trial_ids)}, "
+                f"missing={sorted(trial_ids - session_ids)}; "
+                "see metrics.json and harbor/*/result.json"
+            )
 
 
 def main(argv=None):
 
@@ -0,0 +1,80 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Per-trial KV sessions for Harbor's unchanged mini-SWE-agent loop."""
+
+import asyncio
+import logging
+from urllib.parse import quote
+
+import httpx
+import yaml
+
+from harbor.agents.installed.mini_swe_agent import MiniSweAgent
+
+RETRYABLE_STATUS_CODES = {408, 409, 429, 500, 502, 503, 504}
+
+
+class SessionMiniSweAgent(MiniSweAgent):
+    async def run(self, instruction, environment, context):
+        if self.context_id is None:
+            raise ValueError("Harbor must assign the agent's trial context_id")
+        config = yaml.safe_load(self._config_yaml) if self._config_yaml else {}
+        headers = (
+            config.setdefault("model", {})
+            .setdefault("model_kwargs", {})
+            .setdefault("extra_headers", {})
+        )
+        headers["X-ExecuTorch-Session-ID"] = str(self.context_id)
+        self._config_yaml = yaml.safe_dump(config, sort_keys=False)
+        await super().run(instruction, environment, context)
+
+
+class SessionCleanup:
+    def __init__(self, base_url: str):
+        self.base_url = base_url.rstrip("/")
+
+    async def on_job_start(self, job):
+        # Harbor awaits END hooks before releasing the trial's concurrency slot.
+        job.on_trial_ended(self.on_trial_end)
+
+    async def on_trial_end(self, event):
+        session_id = quote(str(event.trial_id), safe="")
+        try:
+            async with (
+                asyncio.timeout(120),
+                httpx.AsyncClient(trust_env=False, timeout=20) as client,
+            ):
+                delay = 1
+                while True:
+                    try:
+                        response = await client.delete(
+                            f"{self.base_url}/sessions/{session_id}"
+                        )
+                    except httpx.TransportError as error:
+                        reason = str(error)
+                    else:
+                        if response.status_code not in RETRYABLE_STATUS_CODES:
+                            response.raise_for_status()
+                            return
+                        reason = f"HTTP {response.status_code}"
+                    logging.getLogger(__name__).warning(
+                        "Retrying session cleanup for trial %s (%s) in %ss: %s",
+                        event.trial_name,
+                        session_id,
+                        delay,
+                        reason,
+                    )
+                    await asyncio.sleep(delay)
+                    delay = min(delay * 2, 5)
+        except (httpx.HTTPError, TimeoutError) as error:
+            raise RuntimeError(
+                f"Could not release session {session_id} for trial {event.trial_name}; "
+                "aborting evaluation to avoid silent session leaks"
+            ) from error
+
+    async def on_job_end(self, job_result):
+        pass