fix(executors): keep the inherited launch paths at every prepend site

src/hyperloom/inference_optimizer/tests/test_framework_rewrite_evidence.py

@@ -1386,13 +1386,58 @@ def test_probe_injection_is_idempotent(tmp_path, monkeypatch):
     assert entries.count(str(evidence.probe_asset_dir())) == 1
 
 
+def _injected_pythonpath(tmp_path: Path, envs: dict[str, Any]) -> str:
+    """Arm the start-up shim on a fresh config carrying ``envs`` and return the resulting PYTHONPATH."""
+    import yaml
+
+    from hyperloom.orchestrator.actions.executors.profile import ProfileExecutor
+
+    config = tmp_path / "profile.yaml"
+    _write_profile_config(config, envs)
+    ProfileExecutor()._inject_host_probe(config, tmp_path / "ws")
+    return yaml.safe_load(config.read_text(encoding="utf-8"))["benchmark"]["envs"]["PYTHONPATH"]
+
+
+def test_probe_injection_keeps_the_inherited_pythonpath(tmp_path, monkeypatch):
+    """Magpie replaces the launch PYTHONPATH with the config's, so a config without one carries the inherited one."""
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/a", "/b"]))
+    shim = str(evidence.probe_asset_dir())
+    assert _injected_pythonpath(tmp_path, {"TP": 8}) == os.pathsep.join([shim, "/a", "/b"])
+
+
+def test_probe_injection_does_not_append_the_inherited_pythonpath_to_the_config_one(tmp_path, monkeypatch):
+    """A config that carries PYTHONPATH is authoritative over the inherited value."""
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/a", "/b"]))
+    shim = str(evidence.probe_asset_dir())
+    assert _injected_pythonpath(tmp_path, {"PYTHONPATH": "/c"}) == os.pathsep.join([shim, "/c"])
+
+
+def test_probe_injection_moves_a_present_shim_to_the_front(tmp_path, monkeypatch):
+    """Python imports only the first sitecustomize, so a shim already on the path but not first is moved up."""
+    import yaml
+
+    from hyperloom.orchestrator.actions.executors.profile import ProfileExecutor
+
+    monkeypatch.delenv("PYTHONPATH", raising=False)
+    shim = str(evidence.probe_asset_dir())
+    config = tmp_path / "profile.yaml"
+    _write_profile_config(config, {"PYTHONPATH": os.pathsep.join(["/c", shim, "", "/d"])})
+    executor = ProfileExecutor()
+    executor._inject_host_probe(config, tmp_path / "ws")
+    first = yaml.safe_load(config.read_text(encoding="utf-8"))["benchmark"]["envs"]["PYTHONPATH"]
+    assert first == os.pathsep.join([shim, "/c", "/d"])
+    executor._inject_host_probe(config, tmp_path / "ws")
+    assert yaml.safe_load(config.read_text(encoding="utf-8"))["benchmark"]["envs"]["PYTHONPATH"] == first
+
+
 def test_probe_injection_respects_the_off_switch(tmp_path, monkeypatch):
     """With the probe switched off only the start-up shim goes in, without the probe's environment."""
     import yaml
 
     from hyperloom.orchestrator.actions.executors.profile import ProfileExecutor
 
     monkeypatch.setenv(evidence.ENABLE_ENV, "0")
+    monkeypatch.delenv("PYTHONPATH", raising=False)
     config = tmp_path / "profile.yaml"
     _write_profile_config(config, {"TP": 8})
     assert ProfileExecutor()._inject_host_probe(config, tmp_path / "ws") == ""

src/hyperloom/inference_optimizer/tests/test_grid_runner_helpers_coverage_unit.py

@@ -337,6 +337,28 @@ def test_build_variant_yaml_prepends_legit_overlay(tmp_path: Path) -> None:
     assert envs.get("PYTHONPATH", "").startswith(str(overlay))
 
 
+@pytest.mark.parametrize(
+    ("config_envs", "tail"),
+    [({}, ["/inh/a", "/inh/b"]), ({"PYTHONPATH": "/cfg"}, ["/cfg"])],
+    ids=["inherited", "config-authoritative"],
+)
+def test_build_variant_yaml_overlay_builds_on_the_launch_pythonpath(tmp_path, monkeypatch, config_envs, tail) -> None:
+    # The launch overlays benchmark.envs on os.environ, so a config without PYTHONPATH carries the inherited one.
+    # The overlay is relative so the path carries no drive colon on any host.
+    import os
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"]))
+    (tmp_path / "overlay").mkdir()
+    base = tmp_path / "base.yaml"
+    base.write_text(yaml.safe_dump({"benchmark": {"framework": "sglang", "envs": config_envs}}), encoding="utf-8")
+    variant = _variant("v1")
+    variant.overlay_pythonpath = "overlay"  # type: ignore[attr-defined]
+    out = gr._build_variant_yaml(base, "", variant, output_subdir=tmp_path / "v1")
+    envs = yaml.safe_load(out.read_text(encoding="utf-8"))["benchmark"]["envs"]
+    assert envs["PYTHONPATH"] == os.pathsep.join(["overlay", *tail])
+
+
 def test_build_variant_yaml_drops_unsafe_overlay(tmp_path: Path) -> None:
     # SWSPLAT-42358: a ``:``-joined / traversal / non-existent overlay is dropped (would otherwise smuggle extra
     # PYTHONPATH entries or escape).

src/hyperloom/inference_optimizer/tests/test_grid_runner_runtime_override.py

@@ -34,17 +34,46 @@ def test_path_prefix_prepended():
 
 
 def test_path_prefix_not_duplicated():
-    envs = {"PATH": "/attempt/bin:/usr/bin"}
+    envs = {"PATH": os.pathsep.join(["/attempt/bin", "/usr/bin"])}
     apply_runtime_override(envs, {"path_prefix": "/attempt/bin"})
     assert envs["PATH"].count("/attempt/bin") == 1
 
 
+def test_path_prefix_moves_a_present_entry_to_the_front():
+    envs = {"PATH": os.pathsep.join(["/usr/bin", "/attempt/bin"])}
+    apply_runtime_override(envs, {"path_prefix": "/attempt/bin"})
+    assert envs["PATH"] == os.pathsep.join(["/attempt/bin", "/usr/bin"])
+
+
+def test_path_prefix_keeps_the_inherited_path(monkeypatch):
+    """The launch overlays ``benchmark.envs`` on ``os.environ``, so a config without PATH must carry the inherited one."""
+    monkeypatch.setenv("PATH", os.pathsep.join(["/usr/local/bin", "/usr/bin"]))
+    envs: dict[str, str] = {}
+    apply_runtime_override(envs, {"path_prefix": "/attempt/bin"})
+    assert envs["PATH"] == os.pathsep.join(["/attempt/bin", "/usr/local/bin", "/usr/bin"])
+
+
 def test_pythonpath_prefix_prepended(tmp_path):
     envs: dict[str, str] = {}
     apply_runtime_override(envs, {"pythonpath_prefix": str(tmp_path)})
     assert str(tmp_path) in envs.get("PYTHONPATH", "")
 
 
+def test_pythonpath_prefix_keeps_the_inherited_pythonpath(monkeypatch):
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"]))
+    envs: dict[str, str] = {}
+    apply_runtime_override(envs, {"pythonpath_prefix": "/attempt/pkg"})
+    assert envs["PYTHONPATH"] == os.pathsep.join(["/attempt/pkg", "/inh/a", "/inh/b"])
+
+
+def test_pythonpath_prefix_prepends_to_the_config_pythonpath(monkeypatch):
+    """A config that carries PYTHONPATH is authoritative over the inherited value."""
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"]))
+    envs = {"PYTHONPATH": "/cfg"}
+    apply_runtime_override(envs, {"pythonpath_prefix": "/attempt/pkg"})
+    assert envs["PYTHONPATH"] == os.pathsep.join(["/attempt/pkg", "/cfg"])
+
+
 def test_pythonpath_unsafe_colon_dropped():
     envs: dict[str, str] = {}
     apply_runtime_override(envs, {"pythonpath_prefix": "/a:/b"})
@@ -125,28 +154,46 @@ def test_no_runtime_override_field_is_noop(tmp_path):
 # compiled-artifact runtime prefixes
 
 
-def test_pythonpath_prefixes_multi_entry_ordered_prepended():
+def test_pythonpath_prefixes_multi_entry_ordered_prepended(monkeypatch):
+    monkeypatch.setenv("PYTHONPATH", "/inh/a")
     envs = {"PYTHONPATH": "/opt/venv"}
     apply_runtime_override(envs, {"pythonpath_prefixes": ["/a/pkg", "/b/pkg"]})
-    assert envs["PYTHONPATH"] == "/a/pkg:/b/pkg:/opt/venv"
+    assert envs["PYTHONPATH"] == os.pathsep.join(["/a/pkg", "/b/pkg", "/opt/venv"])
 
 
-def test_pythonpath_prefixes_drops_unsafe_entry():
+def test_pythonpath_prefixes_keep_the_inherited_pythonpath(monkeypatch):
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/inh/a", "/inh/b"]))
+    envs: dict[str, str] = {}
+    apply_runtime_override(envs, {"pythonpath_prefixes": ["/a/pkg", "/b/pkg"]})
+    assert envs["PYTHONPATH"] == os.pathsep.join(["/a/pkg", "/b/pkg", "/inh/a", "/inh/b"])
+
+
+def test_pythonpath_prefixes_drops_unsafe_entry(monkeypatch):
+    monkeypatch.delenv("PYTHONPATH", raising=False)
     envs = {}
     apply_runtime_override(envs, {"pythonpath_prefixes": ["/good", "../evil"]})
     assert envs.get("PYTHONPATH") == "/good"
 
 
-def test_ld_library_path_prefix_prepends_and_preserves_rocm():
+def test_ld_library_path_prefix_prepends_and_preserves_rocm(monkeypatch):
+    monkeypatch.setenv("LD_LIBRARY_PATH", "/inh/lib")
     envs = {"LD_LIBRARY_PATH": "/opt/rocm/lib"}
     apply_runtime_override(envs, {"ld_library_path_prefix": ["/attempt/lib"]})
-    assert envs["LD_LIBRARY_PATH"] == "/attempt/lib:/opt/rocm/lib"
+    assert envs["LD_LIBRARY_PATH"] == os.pathsep.join(["/attempt/lib", "/opt/rocm/lib"])
+
+
+def test_ld_library_path_prefix_keeps_the_inherited_rocm_lib(monkeypatch):
+    monkeypatch.setenv("LD_LIBRARY_PATH", "/opt/rocm/lib")
+    envs: dict[str, str] = {}
+    apply_runtime_override(envs, {"ld_library_path_prefix": ["/attempt/lib"]})
+    assert envs["LD_LIBRARY_PATH"] == os.pathsep.join(["/attempt/lib", "/opt/rocm/lib"])
 
 
-def test_ld_library_path_prefix_multi_entry_order():
+def test_ld_library_path_prefix_multi_entry_order(monkeypatch):
+    monkeypatch.delenv("LD_LIBRARY_PATH", raising=False)
     envs = {}
     apply_runtime_override(envs, {"ld_library_path_prefix": ["/a", "/b"]})
-    assert envs["LD_LIBRARY_PATH"] == "/a:/b"
+    assert envs["LD_LIBRARY_PATH"] == os.pathsep.join(["/a", "/b"])
 
 
 def test_runtime_env_merged():
@@ -162,10 +209,18 @@ def test_runtime_env_rejects_reserved_and_blocked_keys():
     assert envs == {"OK_KEY": "v"}
 
 
-def test_entrypoint_bin_dir_prepended_to_path():
+def test_entrypoint_bin_dir_prepended_to_path(monkeypatch):
+    monkeypatch.setenv("PATH", "/inh/bin")
     envs = {"PATH": "/usr/bin"}
     apply_runtime_override(envs, {"entrypoint_bin_dir": "/attempt/bin"})
-    assert envs["PATH"] == "/attempt/bin:/usr/bin"
+    assert envs["PATH"] == os.pathsep.join(["/attempt/bin", "/usr/bin"])
+
+
+def test_entrypoint_bin_dir_keeps_the_inherited_path(monkeypatch):
+    monkeypatch.setenv("PATH", os.pathsep.join(["/usr/local/bin", "/usr/bin"]))
+    envs: dict[str, str] = {}
+    apply_runtime_override(envs, {"entrypoint_bin_dir": "/attempt/bin"})
+    assert envs["PATH"] == os.pathsep.join(["/attempt/bin", "/usr/local/bin", "/usr/bin"])
 
 
 def test_entrypoint_bin_dir_unsafe_dropped():

src/hyperloom/inference_optimizer/tests/test_workload_envs_branches_unit.py

@@ -623,6 +623,75 @@ def test_profile_manual_max_iters_above_cap_warns(monkeypatch, tmp_path, caplog)
     assert any("exceeds the serialization-safe cap" in r.message for r in caplog.records)
 
 
+def _arm_sglang_shape_tool(monkeypatch, tmp_path) -> Path:
+    """Route the SGLang profile through the kernel_shape_tool path and return the tool dir."""
+    from hyperloom.orchestrator.actions.executors import _server_patcher
+
+    tool_dir = tmp_path / "kernel_shape_tool"
+    tool_dir.mkdir()
+    _clear_env(monkeypatch)
+    monkeypatch.setenv("INFERENCE_OPTIMIZER_DISABLE_TP_CLAMP", "1")
+    monkeypatch.setenv("HYPERLOOM_ENABLE_PATCH", "0")
+    monkeypatch.delenv("HYPERLOOM_PROFILE_SHAPE_DISCOVERY", raising=False)
+    monkeypatch.setattr(_server_patcher, "resolve_sglang_shape_mode", lambda: "sitecustomize")
+    monkeypatch.setattr(_server_patcher, "kernel_shape_tool_dir", lambda: tool_dir)
+    return tool_dir
+
+
+def test_profile_shape_tool_keeps_the_inherited_pythonpath(monkeypatch, tmp_path):
+    # Magpie replaces the launch env's PYTHONPATH with the config's, so a config without one must carry the
+    # inherited entries, or the profile server loses the operator's and the image's import paths.
+    import os
+
+    from hyperloom.orchestrator.actions.executors import _framework_rewrite_evidence as evidence
+    from hyperloom.orchestrator.actions.executors.profile import ProfileExecutor
+
+    tool_dir = _arm_sglang_shape_tool(monkeypatch, tmp_path)
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/a", "/b"]))
+    src = _write(tmp_path / "cfg.yaml", envs={"PROFILE": "1"})
+    out = we.materialize_config_with_envs(src, tmp_path / "out")
+    envs = yaml.safe_load(out.read_text())["benchmark"]["envs"]
+    assert envs["TRACELENS_SHAPE_DISCOVERY"] == "1"
+    assert envs["PYTHONPATH"] == os.pathsep.join([str(tool_dir), "/a", "/b"])
+
+    # The profile shim lands in front of the tool, keeping the inherited tail.
+    ProfileExecutor()._inject_host_probe(out, tmp_path / "ws")
+    envs = yaml.safe_load(out.read_text())["benchmark"]["envs"]
+    assert envs["PYTHONPATH"] == os.pathsep.join([str(evidence.probe_asset_dir()), str(tool_dir), "/a", "/b"])
+
+
+def test_profile_shape_tool_prepends_to_the_config_pythonpath(monkeypatch, tmp_path):
+    # A config that carries PYTHONPATH is authoritative: the inherited value is not appended to it.
+    import os
+
+    tool_dir = _arm_sglang_shape_tool(monkeypatch, tmp_path)
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/a", "/b"]))
+    src = _write(tmp_path / "cfg.yaml", envs={"PROFILE": "1", "PYTHONPATH": "/c"})
+    bench = _materialize(src, tmp_path / "out")
+    assert bench["envs"]["PYTHONPATH"] == os.pathsep.join([str(tool_dir), "/c"])
+
+
+@pytest.mark.parametrize(
+    ("config_envs", "tail"),
+    [({}, ["/a", "/b"]), ({"PYTHONPATH": "/c"}, ["/c"])],
+    ids=["inherited", "config-authoritative"],
+)
+def test_reference_overlay_builds_on_the_launch_pythonpath(monkeypatch, tmp_path, config_envs, tail):
+    # The reference overlay is prepended like every launch path entry: on the inherited value when the config has
+    # no PYTHONPATH, on the config's otherwise. The overlay is relative so it carries no drive colon on any host.
+    import os
+
+    _clear_env(monkeypatch)
+    (tmp_path / "overlay").mkdir()
+    (tmp_path / "overlay" / "sitecustomize.py").write_text("", encoding="utf-8")
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(we, "resolve_reference_launch", lambda: ("", {}, {"overlay_pythonpath": "overlay"}))
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join(["/a", "/b"]))
+    src = _write(tmp_path / "cfg.yaml", envs=config_envs)
+    bench = _materialize(src, tmp_path / "out")
+    assert bench["envs"]["PYTHONPATH"] == os.pathsep.join(["overlay", *tail])
+
+
 def test_quality_ref_variant_compares(monkeypatch, tmp_path):
     # A non-baseline scriptable variant must COMPARE against the operator reference and must NOT write.
     _clear_env(monkeypatch)

src/hyperloom/orchestrator/actions/executors/_grid_runner.py

@@ -323,14 +323,6 @@
     )
 
 
-def _prepend_path_entry(envs: dict[str, str], var: str, entry: str) -> None:
-    """Prepend a single already-validated entry onto a ``:``-joined env var."""
-    parts = [p for p in str(envs.get(var, "") or "").split(":") if p]
-    if entry not in parts:
-        parts.insert(0, entry)
-    envs[var] = ":".join(parts)
-
-
 # Reserved keys carried by dedicated override fields; never accepted via runtime_env.
 _RUNTIME_ENV_RESERVED: frozenset[str] = frozenset({"PATH", "PYTHONPATH", "LD_LIBRARY_PATH"})
 
@@ -339,35 +331,34 @@
     """Inject an attempt runtime override into the materialized YAML envs dict."""
     if not override:
         return
+    from ._workload_envs import prepend_launch_path
+
     path_prefix = str(override.get("path_prefix") or "").strip()
     if path_prefix:
-        _prepend_path_entry(envs, "PATH", path_prefix)
+        prepend_launch_path(envs, "PATH", path_prefix)
     entrypoint_bin = str(override.get("entrypoint_bin_dir") or "").strip()
     if entrypoint_bin:
         if _is_safe_path_entry(entrypoint_bin):
-            _prepend_path_entry(envs, "PATH", entrypoint_bin)
+            prepend_launch_path(envs, "PATH", entrypoint_bin)
         else:
             log.warning("apply_runtime_override: dropping unsafe entrypoint_bin_dir %r", entrypoint_bin)
     pp_prefix = str(override.get("pythonpath_prefix") or "").strip()
     if pp_prefix:
         if _is_safe_path_entry(pp_prefix):
-            _cur_pp = str(envs.get("PYTHONPATH", "") or "")
-            envs["PYTHONPATH"] = f"{pp_prefix}:{_cur_pp}" if _cur_pp else pp_prefix
+            prepend_launch_path(envs, "PYTHONPATH", pp_prefix)
         else:
             log.warning("apply_runtime_override: dropping unsafe pythonpath_prefix %r", pp_prefix)
     # Multi-entry prefixes: prepend in order so the first entry wins, ahead of any single-dir pythonpath_prefix and
     # the inherited value.
     for entry in reversed(_coerce_prefix_list(override.get("pythonpath_prefixes"))):
         if _is_safe_path_entry(entry):
-            _cur_pp = str(envs.get("PYTHONPATH", "") or "")
-            envs["PYTHONPATH"] = f"{entry}:{_cur_pp}" if _cur_pp else entry
+            prepend_launch_path(envs, "PYTHONPATH", entry)
         else:
             log.warning("apply_runtime_override: dropping unsafe pythonpath entry %r", entry)
-    # Native loader path: prepend attempt entries while preserving any inherited entries (e.g. /opt/rocm/lib), which
-    # _prepend_path_entry never drops.
+    # Native loader path: prepend attempt entries while preserving any inherited entries (e.g. /opt/rocm/lib).
     for entry in reversed(_coerce_prefix_list(override.get("ld_library_path_prefix"))):
         if _is_safe_path_entry(entry):
-            _prepend_path_entry(envs, "LD_LIBRARY_PATH", entry)
+            prepend_launch_path(envs, "LD_LIBRARY_PATH", entry)
         else:
             log.warning("apply_runtime_override: dropping unsafe ld_library_path entry %r", entry)
     _runtime_env = override.get("runtime_env")
@@ -474,7 +465,7 @@
         envs.pop(str(k), None)
     for k, v in variant.extra_envs.items():
         envs[str(k)] = str(v)
-    from ._workload_envs import pin_mlperf_round_concurrency
+    from ._workload_envs import pin_mlperf_round_concurrency, prepend_launch_path
 
     pin_mlperf_round_concurrency(envs)
     # The launcher re-exports these unconditionally, so a value carried here is
@@ -496,8 +487,7 @@
             and Path(_overlay).is_dir()
         )
         if _overlay_ok:
-            _cur_pp = str(envs.get("PYTHONPATH", "") or "")
-            envs["PYTHONPATH"] = f"{_overlay}:{_cur_pp}" if _cur_pp else _overlay
+            prepend_launch_path(envs, "PYTHONPATH", _overlay)
         else:
             log.warning(
                 "grid: dropping unsafe overlay_pythonpath %r (not a single "