feat(evals): a NanoClaw image, the controller's failure reason, and evaluations on GKE

.github/workflows/ci.yml

@@ -37,6 +37,7 @@ jobs:
         run: |
           pnpm nx run @moltzap/simulator:local-profile-check
           pnpm nx run @moltzap/simulator:gke-profile-check
+          pnpm nx run @moltzap/simulator:nanoclaw-image-check
           pnpm nx run @moltzap/evals:phoenix-terraform-check
       - run: pnpm typecheck
       - run: pnpm lint

docs/modules/simulator/src.mdx

@@ -738,6 +738,29 @@ export class IncompleteLedgerReceipt extends Schema.TaggedClass<IncompleteLedger
 
 Physical receipt retained when ledger completion could not be published.
 
+### [`isEntryModule`](https://github.com/chughtapan/moltzap/blob/main/packages/simulator/src/cluster/entry.ts#L31)
+
+_Function_
+
+```ts
+export function isEntryModule(moduleUrl: string, invoked?: string): boolean
+```
+
+Whether a module is the process entry point rather than an ordinary import.
+
+Both sides are canonicalized because they are not the same kind of path:
+Node resolves a module's real path before it becomes `import.meta.url`, while
+`process.argv[1]` is whatever the caller typed. Every executable in this
+package reaches its module through a symlink in the controller image, where
+`/opt/moltzap/dist` points at the installed package directory. Comparing the
+two without canonicalizing makes a directly invoked entry point look like an
+import, so the process exits successfully having done nothing.
+
+`realPath` returns undefined for a path that does not exist, so a missing or
+deleted `argv[1]` is a plain false rather than a thrown ENOENT.
+
+**Returns:** Whether both locations name the same real file.
+
 ### [`LedgerFailure`](https://github.com/chughtapan/moltzap/blob/main/packages/simulator/src/ledger/append.ts#L57)
 
 _TypeAlias_
@@ -1397,6 +1420,7 @@ Stable persisted identity for an event class.
 ## Files
 
 - `cluster.ts`
+- `entry.ts`
 - `definition.ts`
 - `catalog.ts`
 - `core.ts`

packages/evals/README.md

@@ -112,6 +112,21 @@ attempt per case-condition cell. Both images below must be immutable lowercase
 - `MOLTZAP_NANOCLAW_IMAGE` is the distinct NanoClaw application image that
   implements the shipped NanoClaw container entrypoint and gateway contract.
 
+Both are produced rather than looked up. Each producer prints one JSON line;
+`pinnedImage` is the digest-pinned reference these variables accept, and
+`--repository` names a registry repository for a run that has to push:
+
+```bash
+node packages/simulator/scripts/build-controller-image.mjs
+node packages/simulator/scripts/build-nanoclaw-image.mjs
+```
+
+The NanoClaw image bundles a pinned upstream NanoClaw checkout, the workspace
+MoltZap channel, and the bootstrap entrypoint the simulator's container runtime
+executes. NanoClaw runs every agent turn in a container it spawns itself and
+refuses to start when no runtime answers, so a NanoClaw cell also needs a
+reachable container runtime (`DOCKER_HOST`); the image carries the client only.
+
 Create the local cluster with an absolute artifact directory as described in
 the [local simulator profile](../simulator/local/README.md), then pass that same
 directory to the evaluation process:
@@ -131,9 +146,22 @@ MOLTZAP_TEMPORAL_ADDRESS=127.0.0.1:7233 \
   --nanoclaw-model "$NANOCLAW_MODEL"
 ```
 
-For GKE, use the [GKE simulator profile](../simulator/gke/README.md), push the
-controller/support image to its registry, authenticate `gcloud` for artifact
-readback, and provide the selected cluster and retained bucket:
+For GKE, prefer the profile's own verb, which publishes the controller image,
+holds the supervised Temporal port-forward, and derives every cluster identity
+below from the cluster it just attached to:
+
+```bash
+OPENAI_API_KEY=... ANTHROPIC_API_KEY=... MOLTZAP_NANOCLAW_IMAGE=... \
+packages/simulator/gke/cluster.sh evals \
+  --report-id baseline-2026-08-06 \
+  --openclaw-model "$OPENCLAW_MODEL" \
+  --nanoclaw-model "$NANOCLAW_MODEL"
+```
+
+To drive it by hand instead, use the
+[GKE simulator profile](../simulator/gke/README.md), push the controller/support
+image to its registry, authenticate `gcloud` for artifact readback, and provide
+the selected cluster and retained bucket:
 
 ```bash
 OPENAI_API_KEY=... \

packages/evals/src/cli.test.ts

@@ -0,0 +1,148 @@
+import { assert, effect, it } from "@effect/vitest";
+import { FileSystem } from "@effect/platform";
+import { NodeContext } from "@effect/platform-node";
+import { fileURLToPath } from "node:url";
+import { DateTime, Effect, Schema } from "effect";
+import { IncompleteLedgerReceipt } from "@moltzap/simulator";
+import { ledgerRef } from "@moltzap/simulator/ledger";
+import { evaluationCase } from "./cases.js";
+import {
+  invalidImageDetail,
+  ledgerAllocationFailed,
+  missingImageDetail,
+  runInfrastructureFailed,
+  type AttemptContext,
+  type EvaluationImageKey,
+} from "./cli.js";
+import {
+  decodeEvaluationCaseId,
+  decodeCriterionId,
+  decodeConditionId,
+} from "./model.js";
+import {
+  EvaluationCasePlan,
+  EvaluationConditionPlan,
+  decodeEvaluationAttemptId,
+} from "./sweep.js";
+
+const CASE_ID = decodeEvaluationCaseId("EVAL-006");
+const DIAGNOSTIC =
+  "controller Job failed\nBackoffLimitExceeded: Job has reached the backoff limit";
+
+// The real bundled case, so a catalog rename fails here rather than leaving the
+// attempt vocabulary pinned against a fixture nothing executes.
+function attemptContext(): AttemptContext {
+  const definition = evaluationCase(CASE_ID);
+  if (definition === undefined) {
+    throw new Error(`${CASE_ID} is not a bundled evaluation case`);
+  }
+  return {
+    definition,
+    startedAt: DateTime.unsafeMake(0),
+    cell: {
+      attemptId: decodeEvaluationAttemptId("eval-006-nanoclaw-1"),
+      casePlan: EvaluationCasePlan.make({
+        id: CASE_ID,
+        definitionId: definition.definitionId,
+        name: CASE_ID,
+        description: `Deterministic ${CASE_ID} fixture.`,
+        rubric: `Assess ${CASE_ID}.`,
+        criterionIds: [decodeCriterionId(`${CASE_ID}.result/v1`)],
+        slices: ["baseline"],
+      }),
+      conditionPlan: EvaluationConditionPlan.make({
+        id: decodeConditionId("nanoclaw/v2"),
+        runtimeName: "nanoclaw",
+        runtimeConfiguration: { mode: "deterministic" },
+      }),
+      sample: 1,
+    },
+  };
+}
+
+const RECEIPT = IncompleteLedgerReceipt.make({
+  ledger: Schema.decodeSync(ledgerRef)("eval-006-nanoclaw-1"),
+});
+
+// An operator whose environment is missing an image has no other way to learn
+// that the reference is produced rather than looked up, and the NanoClaw image
+// has no producer anywhere else in the repository.
+effect.each([
+  ["MOLTZAP_CONTROLLER_IMAGE", "build-controller-image.mjs"],
+  ["MOLTZAP_SUPPORT_IMAGE", "build-controller-image.mjs"],
+  ["MOLTZAP_NANOCLAW_IMAGE", "build-nanoclaw-image.mjs"],
+] as const)(
+  "names the producer of %s in both of its configuration failures",
+  ([key, script]: readonly [EvaluationImageKey, string]) =>
+    Effect.gen(function* () {
+      const fileSystem = yield* FileSystem.FileSystem;
+      const producer = `packages/simulator/scripts/${script}`;
+
+      for (const detail of [missingImageDetail(key), invalidImageDetail(key)]) {
+        assert.include(detail, key);
+        assert.include(detail, producer);
+        assert.include(detail, "pinnedImage");
+      }
+
+      // A named script that does not exist is worse than no remedy at all.
+      assert.isTrue(
+        yield* fileSystem.exists(
+          fileURLToPath(new URL(`../../../${producer}`, import.meta.url)),
+        ),
+        `${producer} does not exist`,
+      );
+    }).pipe(Effect.provide(NodeContext.layer)),
+);
+
+// Both infrastructure attempts already carry the operator-facing account of a
+// failure, and phoenix-run publishes exactly that field as the run's error, so
+// the controller's own account belongs in it rather than beside it.
+effect.each([undefined, DIAGNOSTIC])(
+  "gives a failed run the controller account %s",
+  (diagnostic?: string) =>
+    Effect.gen(function* () {
+      const attempt = yield* runInfrastructureFailed(
+        attemptContext(),
+        RECEIPT,
+        diagnostic,
+      );
+
+      assert.strictEqual(attempt.detail, diagnostic ?? attempt.detail);
+      assert.isNotEmpty(attempt.detail);
+      if (diagnostic === undefined) {
+        assert.include(attempt.detail, "infrastructure failure");
+      }
+    }),
+);
+
+effect.each([undefined, DIAGNOSTIC])(
+  "gives a failed allocation the controller account %s",
+  (diagnostic?: string) =>
+    Effect.gen(function* () {
+      const attempt = yield* ledgerAllocationFailed(
+        attemptContext(),
+        diagnostic,
+      );
+
+      assert.strictEqual(
+        attempt.failure.detail,
+        diagnostic ?? attempt.failure.detail,
+      );
+      assert.isNotEmpty(attempt.failure.detail);
+      if (diagnostic === undefined) {
+        assert.include(attempt.failure.detail, "durable ledger");
+      }
+    }),
+);
+
+// Without a controller account the two attempts still have to be told apart:
+// a run that never got a ledger and a run that lost its cluster are different
+// operator problems, and the fallback text is all that says which happened.
+it("distinguishes the two infrastructure failures when neither left an account", () =>
+  Effect.gen(function* () {
+    const context = attemptContext();
+    const failedRun = yield* runInfrastructureFailed(context, RECEIPT);
+    const failedAllocation = yield* ledgerAllocationFailed(context);
+
+    assert.notStrictEqual(failedRun.detail, failedAllocation.failure.detail);
+  }));