@@ -176,3 +176,4 @@ pyrightconfig.json
 # End of https://www.toptal.com/developers/gitignore/api/python
 /jobs/
 /experiments/
+.idea
@@ -45,8 +45,18 @@ uv run programbench harbor export ./out --filter '^svenstaro__' --slice 0:5
 uv run programbench harbor export ./out --target-language Kotlin <id>
 # Restrict the agent's network allowlist to a specific model API host
 uv run programbench harbor export ./out <id> --allowed-host api.openai.com
+# Export an oracle-verifiable task (the golden solve.sh needs root; see below)
+uv run programbench harbor export ./out <id> --agent-user root
 ```
 
+The agent phase runs as the cleanroom image's non-root `agent` user, so the
+mode-111 reference binary at `/workspace/executable` can be executed but not
+read, copied, or statically inspected; the generated agent image also drops the
+image's NOPASSWD sudo grant, which would otherwise hand root back. The agent's
+network allowlist covers the model API only — Harbor installs the agent CLI
+during *setup*, which runs before the agent-phase policy applies, so no
+package-registry or source-hosting host belongs on that list.
+
 ## Language comparison experiment
 
 `scripts/language_comparison/run_language_comparison.sh` reads the languages,
@@ -69,8 +79,14 @@ comparisons of reward, cost, agent steps, and tokens.
 
 The config must contain exactly two languages, one agent, positive parallelism,
 and an explicit `tasks.task_names` list. `parallelism` controls concurrent trials
-globally across both languages. Report deltas are the second language minus the
-first.
+globally across both languages; keep `parallelism * harbor.DOCKER_CPUS` inside
+the host's core count. Trials are submitted with the languages interleaved task
+by task, so drift in host load or model serving over a long run cannot align with
+the language contrast. Each run also resolves and pins one agent CLI version for
+all of its trials, recorded in `manifest.json` and the derived job config; set
+`agent.kwargs.version` yourself to skip the registry lookup. Report deltas are
+the second language minus the first, and `report.md` carries a paired-difference
+interval over the per-task deltas.
 Each experiment keeps its generated tasks, derived configs, Harbor jobs,
 `manifest.json`, `report.md`, and `report.json` under its output directory.
 
@@ -96,9 +112,10 @@ Each instance produces a standard Harbor task directory:
 
 ProgramBench's environment ships no source, but a working reference build sits
 at the canonical path the tests target — `/workspace/executable` — execute-only
-so a non-root agent can't read its bytes. The oracle (which runs as root)
-reconstructs `./executable` from it with **no network** and no per-language
-vendoring:
+so a non-root agent can't read its bytes. The oracle reads them, so it needs
+root: Harbor runs `solve.sh` as the task's `[agent] user`, which means oracle
+verification requires a task exported with `--agent-user root`. It then
+reconstructs `./executable` with **no network** and no per-language vendoring:
 
 1. `solve.sh` base64-encodes `/workspace/executable` into
    `/workspace/.programbench_oracle_reference`.
@@ -137,6 +154,7 @@ if the JUnit XML marks it passed (absent counts as unresolved). It writes:
   still needs network access while building the task images.
 
 ```bash
+uv run programbench harbor export ./out <instance_id> --agent-user root
 harbor trials start --path ./out/<instance_id> --agent oracle
 ```
 
 
@@ -9,6 +9,8 @@
 import typer
 import yaml
 
+from kotlinai.harbor import AGENT_USER
+
 app = typer.Typer(
     name="harbor",
     help="Export and compare tasks in the Harbor task format (harborframework.com).",
@@ -32,6 +34,12 @@ def export(
         "--target-language",
         help="Language the agent must use for its implementation; defaults to the instance language.",
     ),
+    agent_user: str = typer.Option(
+        AGENT_USER,
+        "--agent-user",
+        help="User the agent phase runs as. The default cannot read the execute-only reference binary; "
+        "pass 'root' to export an oracle-verifiable task (the golden solve.sh must read it).",
+    ),
 ) -> None:
     """Convert ProgramBench instances into Harbor tasks under OUT_DIR."""
     from kotlinai.harbor import convert_all
@@ -49,6 +57,7 @@ def _skip(instance_id: str, exc: Exception) -> None:
         slice_spec=slice_spec,
         allowed_hosts=allowed_host or None,
         target_language=target_language,
+        agent_user=agent_user,
         on_error=_skip,
     )
     for p in paths:
 
@@ -5,7 +5,7 @@ FROM {{ image_name }}:{{ image_tag }}
 {% if install_kotlin %}
 RUN sed -i 's#https://snapshot.ubuntu.com/ubuntu/[0-9TZ]*/#http://archive.ubuntu.com/ubuntu/#g' /etc/apt/sources.list \
     && apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl openjdk-21-jdk-headless ripgrep unzip \
+    && apt-get install -y --no-install-recommends ca-certificates curl file openjdk-21-jdk-headless ripgrep unzip xxd \
     && curl -fsSLo /tmp/kotlin-compiler.zip \
         "https://github.com/JetBrains/kotlin/releases/download/v{{ kotlin_version }}/kotlin-compiler-{{ kotlin_version }}.zip" \
     && echo "{{ kotlin_compiler_sha256 }}  /tmp/kotlin-compiler.zip" | sha256sum -c - \
@@ -15,14 +15,20 @@ RUN sed -i 's#https://snapshot.ubuntu.com/ubuntu/[0-9TZ]*/#http://archive.ubuntu
 {% elif install_jdk %}
 RUN sed -i 's#https://snapshot.ubuntu.com/ubuntu/[0-9TZ]*/#http://archive.ubuntu.com/ubuntu/#g' /etc/apt/sources.list \
     && apt-get update \
-    && apt-get install -y --no-install-recommends openjdk-21-jdk-headless ripgrep \
+    && apt-get install -y --no-install-recommends file openjdk-21-jdk-headless ripgrep xxd \
     && rm -rf /var/lib/apt/lists/*
 {% else %}
 RUN sed -i 's#https://snapshot.ubuntu.com/ubuntu/[0-9TZ]*/#http://archive.ubuntu.com/ubuntu/#g' /etc/apt/sources.list \
     && apt-get update \
-    && apt-get install -y --no-install-recommends ripgrep \
+    && apt-get install -y --no-install-recommends file ripgrep xxd \
     && rm -rf /var/lib/apt/lists/*
 {% endif %}
+# The cleanroom image grants the non-root agent NOPASSWD sudo for apt-get, dpkg,
+# pip, npm, cargo and go — each an arbitrary-code-as-root path (e.g. an apt
+# Pre-Invoke hook), which would hand back the ability to read the execute-only
+# reference binary. Nothing legitimate needs it: the toolchains are baked in
+# above and the agent phase has no network.
+RUN rm -f /etc/sudoers.d/agent-packages
 {% if install_jdk %}
 ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
 {% endif %}
 
@@ -1,8 +1,17 @@
 version = "1.0"
-# Hand the agent's workspace to the separate verifier, excluding any prebuilt
-# ./executable so the offline build must reproduce it (matches ProgramBench's
-# submission archive, which the evaluator wipes ./executable from before compile).
-artifacts = [{ source = "/workspace", exclude = ["executable"] }]
+# Hand the agent's workspace to the separate verifier. tests/test.sh drops any
+# prebuilt ./executable before it runs compile.sh, so the offline build still
+# has to reproduce it (matching ProgramBench, whose evaluator wipes ./executable
+# from the submission archive before compiling).
+#
+# Deliberately no `exclude`: Harbor only reaches for its tar-based transfer when
+# an artifact has exclusions, and that path extracts host-side with
+# `tarfile.extractall(filter="data")`, which raises AbsoluteLinkError on any
+# absolute symlink in the tree. An agent aliasing the reference binary
+# (`ln -s /workspace/executable /workspace/refbin`) therefore lost its whole
+# workspace handoff and scored 0. With no exclusions Harbor uses `docker cp`,
+# which is indifferent to symlinks.
+artifacts = ["/workspace"]
 
 [metadata]
 author_name = "ProgramBench"
@@ -15,17 +24,26 @@ upstream_commit = "{{ commit }}"
 {% endif %}solution_explanation = "The reference source is intentionally removed; the agent must reverse-engineer the installed black-box binary and ship a compile.sh that rebuilds it offline."
 verification_explanation = "A separate verifier container (built from the cleanroom image with /workspace wiped) receives the agent's /workspace as an artifact minus ./executable, rebuilds ./executable from compile.sh with no network, then runs each active test branch's behavioral suite against it. Reward is the fraction of non-ignored tests that pass."
 
-# No network during inference: the agent must reverse-engineer the binary, not
-# fetch upstream source. The agent phase overrides the baseline to an
-# LLM-API-only allowlist (requires a dynamic_network_policy environment backend,
-# e.g. Docker/OrbStack on Linux with nftables).
+# The environment baseline stays public so Harbor can build the image and install
+# the agent CLI (agent setup runs before the agent-phase policy applies).
 [environment]
 build_timeout_sec = {{ build_timeout }}
 cpus = {{ env_cpus }}
 network_mode = "public"
 
+# No network during inference beyond the model API: the agent must
+# reverse-engineer the binary, not fetch upstream source. Overriding the baseline
+# per phase requires a dynamic_network_policy environment backend (e.g.
+# Docker/OrbStack on Linux with nftables).
+#
+# The agent runs as {{ agent_user }}. The default is a non-root user, so the
+# mode-111 reference binary at /workspace/executable can be executed but not
+# read, copied, or statically inspected; root would bypass those mode bits
+# entirely and is only for oracle verification, whose golden solve.sh must read
+# the reference (`harbor export --agent-user root`).
 [agent]
 timeout_sec = {{ agent_timeout }}
+user = "{{ agent_user }}"
 network_mode = "allowlist"
 allowed_hosts = {{ allowed_hosts | tojson }}
 
 
@@ -11,9 +11,12 @@
 import copy
 import hashlib
 import json
+import math
 import os
 import re
+import statistics
 import subprocess
+import urllib.request
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
@@ -23,7 +26,7 @@
 
 from kotlinai.harbor import convert_all
 
-REPORT_SCHEMA_VERSION = 2
+REPORT_SCHEMA_VERSION = 3
 RESERVED_HARBOR_FIELDS = {
     "agents",
     "datasets",
@@ -35,10 +38,45 @@
 }
 
 
+# npm packages the supported agent CLIs are published as. Harbor installs
+# `@latest` unless an explicit version is passed, so a release landing mid-run
+# would apply to whichever trials happen to start after it — with two languages
+# in one job that is a confound, not just noise.
+AGENT_VERSION_PACKAGES = {"codex": "@openai/codex"}
+
+# Two-sided 95% Student's t critical values indexed by degrees of freedom (entry
+# 0 is unused). Beyond the table the value has converged on the normal quantile.
+# fmt: off
+T_CRITICAL_95 = (
+    0.0, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228,
+    2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086,
+    2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042,
+)
+# fmt: on
+
+
 class ComparisonError(ValueError):
     """Raised when an experiment cannot guarantee a fair comparison."""
 
 
+def resolve_agent_version(package: str) -> str:
+    """Return the version npm currently publishes as ``latest`` for a package."""
+    with urllib.request.urlopen(f"https://registry.npmjs.org/{package}/latest", timeout=30) as response:
+        return json.load(response)["version"]
+
+
+def pin_agent_version(agent: dict[str, Any], resolver: Callable[[str], str]) -> None:
+    """Pin ``agent`` to one CLI version for the whole run.
+
+    A no-op when there is nothing to pin: the agent is not one we know an npm
+    package for, or the caller already pinned it explicitly.
+    """
+    package = AGENT_VERSION_PACKAGES.get(agent["name"].strip().lower())
+    kwargs = agent.setdefault("kwargs", {})
+    if package is not None and not kwargs.get("version"):
+        kwargs["version"] = resolver(package)
+
+
 @dataclass(frozen=True)
 class ComparisonSettings:
     languages: tuple[str, str]
@@ -140,7 +178,21 @@ def build_comparison_job_config(
     jobs_dir: Path,
     job_name: str,
 ) -> dict[str, Any]:
-    """Create one Harbor job containing every language-specific dataset."""
+    """Create one Harbor job with both languages interleaved task by task.
+
+    One single-task dataset per (task, language) is what interleaves them: Harbor
+    resolves datasets in order and submits trials in that order, so one dataset
+    per language would run every trial of the first language before the second
+    and fold any drift in host load or model serving into the language contrast.
+    The dataset path stays the language directory, so each trial's ``source`` is
+    still the language and the report can attribute it.
+
+    Harbor's ``tasks`` list would express this more directly, but it seeds no
+    entry in the job's metric registry (``Job._resolve_metrics`` walks
+    ``datasets`` only) while the live progress display indexes
+    ``_metrics[source][0]`` — a defaultdict miss that aborts the whole job with
+    an IndexError partway through. Datasets keep that registry populated.
+    """
     config = copy.deepcopy(settings.harbor)
     config["job_name"] = job_name
     config["jobs_dir"] = str(jobs_dir.resolve())
@@ -149,8 +201,9 @@ def build_comparison_job_config(
     config["datasets"] = [
         {
             "path": str(task_dirs[_language_key(language)].resolve()),
-            "task_names": list(settings.task_names),
+            "task_names": [task_name],
         }
+        for task_name in settings.task_names
         for language in settings.languages
     ]
     return config
@@ -184,6 +237,7 @@ def run_comparison_experiment(
     *,
     exporter: Callable[..., list[Path]] = convert_all,
     agent_runner: Callable[[Path, Path], int] | None = None,
+    version_resolver: Callable[[str], str] = resolve_agent_version,
 ) -> int:
     """Export both languages and run all trials in one parallel Harbor job."""
     config_path = config_path.resolve()
@@ -195,6 +249,10 @@ def run_comparison_experiment(
         raise ComparisonError(f"Agent runner does not exist: {runner}")
 
     settings = validate_comparison_config(_read_yaml(config_path))
+    # Fail before spending anything if the registry is unreachable; an unpinned
+    # CLI is the confound this run exists to avoid. Pin explicitly in the config
+    # to run without registry access.
+    pin_agent_version(settings.agent, version_resolver)
     output_dir.mkdir(parents=True)
     jobs_dir = output_dir / "jobs"
     repo_root = runner.parent.parent
@@ -206,7 +264,11 @@ def run_comparison_experiment(
         "base_config": str(config_path),
         "base_config_sha256": _config_digest(config_path),
         "runner": str(runner),
-        "agent": {"name": settings.agent.get("name"), "model_name": settings.agent.get("model_name")},
+        "agent": {
+            "name": settings.agent.get("name"),
+            "model_name": settings.agent.get("model_name"),
+            "version": settings.agent["kwargs"].get("version"),
+        },
         "language_order": list(settings.languages),
         "parallelism": settings.parallelism,
         "task_names": settings.task_names,
@@ -467,6 +529,40 @@ def _delta(second: dict[str, Any], first: dict[str, Any]) -> dict[str, int | flo
     return result
 
 
+def _paired_difference(tasks: list[dict[str, Any]], first_key: str, second_key: str) -> dict[str, Any]:
+    """Summarize the per-task reward differences, second language minus first.
+
+    The interval is a Student's t interval over the task differences. It is
+    descriptive: the tasks are a fixed selection, not a random sample of
+    programming work, and each task contributes one mean per language.
+    """
+    pairs = [
+        (_numeric(task["results"][first_key]["reward_mean"]), _numeric(task["results"][second_key]["reward_mean"]))
+        for task in tasks
+    ]
+    deltas = [second - first for first, second in pairs if first is not None and second is not None]
+    mean = statistics.fmean(deltas) if deltas else None
+    sd = statistics.stdev(deltas) if len(deltas) > 1 else None
+    degrees_of_freedom = len(deltas) - 1
+    half_width = (
+        (T_CRITICAL_95[degrees_of_freedom] if degrees_of_freedom < len(T_CRITICAL_95) else 1.96)
+        * sd
+        / math.sqrt(len(deltas))
+        if sd
+        else None
+    )
+    return {
+        "n": len(deltas),
+        "mean": mean,
+        "sd": sd,
+        "ci_low": mean - half_width if half_width is not None else None,
+        "ci_high": mean + half_width if half_width is not None else None,
+        "wins": sum(delta > 0 for delta in deltas),
+        "losses": sum(delta < 0 for delta in deltas),
+        "ties": sum(delta == 0 for delta in deltas),
+    }
+
+
 def build_comparison_report(experiment_dir: Path) -> dict[str, Any]:
     experiment_dir = experiment_dir.resolve()
     manifest = _read_json(experiment_dir / "manifest.json")
@@ -521,6 +617,7 @@ def build_comparison_report(experiment_dir: Path) -> dict[str, Any]:
         "delta": {"from": first_key, "to": second_key},
         "task_count": len(task_names),
         "tasks": tasks,
+        "paired": _paired_difference(tasks, first_key, second_key),
         "totals": {
             "results": total_results,
             "delta": _delta(total_results[second_key], total_results[first_key]),
@@ -660,8 +757,24 @@ def render_markdown_report(report: dict[str, Any]) -> str:
             f"{_format_value(totals['delta'][field], kind, delta=True)} |"
         )
 
+    paired = report["paired"]
     lines.extend(
         [
+            "",
+            "## Paired difference",
+            "",
+            f"| Metric | {second_name} minus {first_name} |",
+            "|---|---:|",
+            f"| Paired tasks | {_format_value(paired['n'], 'count')} |",
+            f"| Mean difference | {_format_value(paired['mean'], 'reward', delta=True)} |",
+            f"| Standard deviation | {_format_value(paired['sd'], 'reward')} |",
+            f"| Approximate 95% interval | [{_format_value(paired['ci_low'], 'reward', delta=True)}, "
+            f"{_format_value(paired['ci_high'], 'reward', delta=True)}] |",
+            f"| Task wins ({second_name} / {first_name} / tie) | {_format_value(paired['wins'], 'count')} / "
+            f"{_format_value(paired['losses'], 'count')} / {_format_value(paired['ties'], 'count')} |",
+            "",
+            "The interval is descriptive, not inferential: the tasks are a fixed selection rather than a random "
+            "sample of programming work, and each task contributes one mean per language.",
             "",
             "## Token breakdown by task",
             "",