fix(eve): recover expired session sandboxes

.changeset/bright-sandboxes-recover.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Recreate persisted Vercel session sandboxes when their snapshots have expired instead of returning an unusable sandbox handle.

packages/eve/src/execution/sandbox/bindings/vercel-errors.ts

@@ -12,6 +12,21 @@ export function isVercelSnapshotUnavailableError(error: unknown): boolean {
   return false;
 }
 
+export function isVercelSnapshotNotFoundError(error: unknown): boolean {
+  for (const candidate of walkErrorChain(error)) {
+    const status =
+      (candidate as { response?: { status?: number } }).response?.status ??
+      (candidate as { status?: number }).status ??
+      (candidate as { statusCode?: number }).statusCode;
+    const code = (candidate as { json?: { error?: { code?: unknown } } }).json?.error?.code;
+    if (status === 410 && code === "snapshot_not_found") {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 export function isVercelSandboxMissingError(error: unknown): boolean {
   for (const candidate of walkErrorChain(error)) {
     const status =

packages/eve/src/execution/sandbox/bindings/vercel-lookup.ts

@@ -11,6 +11,7 @@ import type {
 
 export async function getNamedVercelSandbox(input: {
   readonly createOptions: VercelCreateOptions;
+  readonly resume?: boolean;
   readonly sandboxModule: VercelModule;
   readonly sandboxName: string;
 }): Promise<VercelSandbox | null> {
@@ -32,12 +33,13 @@ export async function getNamedVercelSandbox(input: {
 
 async function getVercelSandboxGetOptions(input: {
   readonly createOptions: VercelCreateOptions;
+  readonly resume?: boolean;
   readonly sandboxName: string;
 }): Promise<VercelGetOptions> {
   const baseOptions = {
     fetch: getVercelSandboxFetch(input.createOptions),
     name: input.sandboxName,
-    resume: false,
+    resume: input.resume ?? false,
   };
 
   try {

packages/eve/src/execution/sandbox/bindings/vercel-session.ts

@@ -0,0 +1,204 @@
+import { createLogger } from "#internal/logging.js";
+import type {
+  VercelSandboxSessionCreateContext,
+  VercelSandboxSessionCreateOptions,
+} from "#public/sandbox/vercel-sandbox.js";
+import type {
+  CreateVercelSandbox,
+  VercelSandboxCreateParams,
+} from "#execution/sandbox/bindings/vercel-create-sdk.js";
+import {
+  isVercelSandboxMissingError,
+  isVercelSnapshotNotFoundError,
+  isVercelSnapshotUnavailableError,
+} from "#execution/sandbox/bindings/vercel-errors.js";
+import { getNamedVercelSandbox } from "#execution/sandbox/bindings/vercel-lookup.js";
+import type {
+  VercelCreateOptions,
+  VercelModule,
+  VercelSandbox,
+} from "#execution/sandbox/bindings/vercel-sdk-types.js";
+
+const logger = createLogger("sandbox.vercel");
+
+export type ResolveVercelSessionCreateOptions = (
+  context: VercelSandboxSessionCreateContext,
+) => Promise<VercelSandboxSessionCreateOptions> | VercelSandboxSessionCreateOptions;
+
+export interface EnsureSessionInput {
+  readonly createOptions: VercelCreateOptions;
+  readonly createSandbox: CreateVercelSandbox;
+  readonly existingMetadata?: Record<string, unknown>;
+  readonly resolveSessionCreateOptions?: ResolveVercelSessionCreateOptions;
+  readonly sandboxModule: VercelModule;
+  readonly sessionId: string;
+  readonly sessionKey: string;
+  readonly snapshotId?: string;
+  readonly tags: Record<string, string> | undefined;
+}
+
+export interface VercelSandboxSessionCreateResult {
+  readonly created: boolean;
+  readonly sandbox: VercelSandbox;
+}
+
+export class VercelTemplateSnapshotUnavailableError extends Error {
+  constructor(options?: ErrorOptions) {
+    super("template snapshot unavailable during session create", options);
+  }
+
+  static is(error: unknown): error is VercelTemplateSnapshotUnavailableError {
+    return error instanceof VercelTemplateSnapshotUnavailableError;
+  }
+}
+
+export async function ensureSession(
+  input: EnsureSessionInput,
+): Promise<VercelSandboxSessionCreateResult> {
+  const sandboxName = getVercelSandboxName(input.existingMetadata) ?? input.sessionKey;
+  let existing: VercelSandbox | null;
+  try {
+    existing = await getNamedVercelSandbox({
+      createOptions: input.createOptions,
+      resume: true,
+      sandboxModule: input.sandboxModule,
+      sandboxName,
+    });
+  } catch (error) {
+    if (!isVercelSnapshotNotFoundError(error)) {
+      throw error;
+    }
+
+    // The backing snapshot expired, so the persisted filesystem is already
+    // unrecoverable. Delete-then-recreate under the same name mirrors the
+    // SDK's own `Sandbox.getOrCreate` recovery; concurrent creates for one
+    // session key share its race window and surface a name conflict to the
+    // loser.
+    logger.warn("session sandbox snapshot expired; deleting it and creating a replacement", {
+      sandboxName,
+    });
+    const stale = await getNamedVercelSandbox({
+      createOptions: input.createOptions,
+      sandboxModule: input.sandboxModule,
+      sandboxName,
+    });
+    try {
+      await stale?.delete();
+    } catch (deleteError) {
+      if (!isVercelSandboxMissingError(deleteError)) {
+        throw deleteError;
+      }
+    }
+    existing = null;
+  }
+
+  if (existing !== null) {
+    await ensureVercelSandboxTags(existing, input.tags);
+    return { created: false, sandbox: existing };
+  }
+
+  const sessionCreateOptions = await input.resolveSessionCreateOptions?.({
+    session: { id: input.sessionId },
+  });
+  const createParams = createSessionCreateParams(input, sandboxName, sessionCreateOptions);
+  if (input.tags !== undefined) {
+    createParams.tags = input.tags;
+  }
+
+  try {
+    return {
+      created: true,
+      sandbox: await input.createSandbox({
+        createOptions: createParams,
+        sandboxModule: input.sandboxModule,
+      }),
+    };
+  } catch (error) {
+    if (
+      input.snapshotId !== undefined &&
+      (isVercelSnapshotUnavailableError(error) || isVercelSandboxMissingError(error))
+    ) {
+      throw new VercelTemplateSnapshotUnavailableError({ cause: error });
+    }
+    throw error;
+  }
+}
+
+function createSessionCreateParams(
+  input: EnsureSessionInput,
+  sandboxName: string,
+  sessionCreateOptions: VercelSandboxSessionCreateOptions = {},
+): VercelSandboxCreateParams {
+  const createOptions = { ...input.createOptions, ...sessionCreateOptions } as VercelCreateOptions;
+  if (input.snapshotId === undefined) {
+    return withBaseSetupNetworkPolicy({
+      ...createOptions,
+      name: sandboxName,
+      persistent: true,
+    });
+  }
+
+  /*
+   * Strip `source`, `runtime`, and `image` from author-supplied create options
+   * for the template-backed session path. The framework owns the source there,
+   * and a snapshot source is mutually exclusive with both `runtime` and `image`
+   * (the template snapshot already has the eve image baked in).
+   */
+  const {
+    image: _image,
+    runtime: _runtime,
+    source: _source,
+    ...baseSessionCreateOptions
+  } = createOptions as VercelCreateOptions &
+    Partial<Record<"image" | "runtime" | "source", unknown>>;
+
+  return {
+    ...baseSessionCreateOptions,
+    name: sandboxName,
+    persistent: true,
+    source: { snapshotId: input.snapshotId, type: "snapshot" as const },
+  };
+}
+
+function getVercelSandboxName(metadata: Record<string, unknown> | undefined): string | undefined {
+  const sandboxName = metadata?.sandboxName;
+  return typeof sandboxName === "string" ? sandboxName : undefined;
+}
+
+/*
+ * Shared with the template lifecycle in vercel.ts: templates and sessions
+ * apply the same tag reconciliation, and both fresh-create paths start
+ * permissive so framework base setup runs before the author's network
+ * policy applies.
+ */
+export function withBaseSetupNetworkPolicy(
+  createOptions: VercelSandboxCreateParams,
+): VercelSandboxCreateParams {
+  return { ...createOptions, networkPolicy: "allow-all" };
+}
+
+export async function ensureVercelSandboxTags(
+  sandbox: VercelSandbox,
+  tags: Record<string, string> | undefined,
+): Promise<void> {
+  if (tags === undefined || areVercelSandboxTagsEqual(sandbox.tags, tags)) {
+    return;
+  }
+
+  await sandbox.update({ tags });
+}
+
+function areVercelSandboxTagsEqual(
+  current: Record<string, string> | undefined,
+  next: Record<string, string>,
+): boolean {
+  const currentTags = current ?? {};
+  const currentEntries = Object.entries(currentTags);
+  const nextEntries = Object.entries(next);
+
+  if (currentEntries.length !== nextEntries.length) {
+    return false;
+  }
+
+  return nextEntries.every(([key, value]) => currentTags[key] === value);
+}

packages/eve/src/execution/sandbox/bindings/vercel.test.ts

@@ -645,7 +645,7 @@ describe("createVercelSandbox", () => {
     expect(get).toHaveBeenCalledWith({
       fetch: expect.any(Function),
       name: "session-key",
-      resume: false,
+      resume: true,
     });
     expect(create).toHaveBeenCalledTimes(1);
     expect(create.mock.calls[0]?.[0]).toMatchObject({
@@ -1067,14 +1067,101 @@ describe("createVercelSandbox", () => {
     expect(sandboxModule.Sandbox.get).toHaveBeenCalledWith({
       fetch: expect.any(Function),
       name: "persisted-sandbox-name",
-      resume: false,
+      resume: true,
     });
     expect(handle.session).toBeDefined();
 
     const state = await handle.captureState();
     expect(state.metadata).toEqual({ sandboxName: "persisted-sandbox-name" });
   });
 
+  it("replaces a session sandbox when its persisted snapshot is unavailable", async () => {
+    const templateSandbox = createMockSandbox({
+      name: "template-key",
+      snapshotId: "template-snapshot",
+    });
+    const staleSessionSandbox = createMockSandbox({ name: "persisted-sandbox-name" });
+    const replacementSessionSandbox = createMockSandbox({ name: "persisted-sandbox-name" });
+    const snapshotUnavailable = Object.assign(new Error("snapshot_not_found"), {
+      json: { error: { code: "snapshot_not_found" } },
+      response: { status: 410 },
+    });
+    const create = vi.fn().mockResolvedValue(replacementSessionSandbox);
+    const get = vi
+      .fn()
+      .mockImplementation(async ({ name, resume }: { name: string; resume: boolean }) => {
+        if (name === "template-key") return templateSandbox;
+        if (name === "persisted-sandbox-name" && resume) throw snapshotUnavailable;
+        if (name === "persisted-sandbox-name") return staleSessionSandbox;
+        return null;
+      });
+    const sandboxModule = { Sandbox: { create, get } };
+
+    const backend = createTestVercelSandbox({
+      loadSandboxModule: async () => sandboxModule as never,
+    });
+
+    await backend.prewarm({
+      runtimeContext: { appRoot: "/tmp/test-app-root" },
+      seedFiles: [],
+      templateKey: "template-key",
+    });
+
+    const handle = await backend.create({
+      existingMetadata: { sandboxName: "persisted-sandbox-name" },
+      runtimeContext: { appRoot: "/tmp/test-app-root" },
+      sessionKey: "session-key",
+      templateKey: "template-key",
+    });
+
+    expect(staleSessionSandbox.delete).toHaveBeenCalledTimes(1);
+    expect(create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        name: "persisted-sandbox-name",
+        persistent: true,
+        source: { snapshotId: "template-snapshot", type: "snapshot" },
+      }),
+    );
+    expect((await handle.captureState()).metadata).toEqual({
+      sandboxName: "persisted-sandbox-name",
+    });
+  });
+
+  it("does not replace a session sandbox for an unrelated 410 response", async () => {
+    const templateSandbox = createMockSandbox({
+      name: "template-key",
+      snapshotId: "template-snapshot",
+    });
+    const unrelatedGone = Object.assign(new Error("resource gone"), {
+      json: { error: { code: "resource_gone" } },
+      response: { status: 410 },
+    });
+    const create = vi.fn();
+    const get = vi.fn().mockImplementation(async ({ name }: { name: string }) => {
+      if (name === "template-key") return templateSandbox;
+      throw unrelatedGone;
+    });
+    const backend = createTestVercelSandbox({
+      loadSandboxModule: async () => ({ Sandbox: { create, get } }) as never,
+    });
+
+    await backend.prewarm({
+      runtimeContext: { appRoot: "/tmp/test-app-root" },
+      seedFiles: [],
+      templateKey: "template-key",
+    });
+
+    await expect(
+      backend.create({
+        existingMetadata: { sandboxName: "persisted-sandbox-name" },
+        runtimeContext: { appRoot: "/tmp/test-app-root" },
+        sessionKey: "session-key",
+        templateKey: "template-key",
+      }),
+    ).rejects.toThrow("Failed to create sandbox session");
+    expect(create).not.toHaveBeenCalled();
+  });
+
   it("stops the session sandbox on shutdown so no VM outlives the server", async () => {
     const { handle, sessionSandbox } = await createTestVercelSession();
 
@@ -1173,7 +1260,7 @@ describe("createVercelSandbox", () => {
     expect(sandboxModule.Sandbox.get).toHaveBeenCalledWith({
       fetch: expect.any(Function),
       name: "deleted-sandbox",
-      resume: false,
+      resume: true,
     });
     expect(sandboxModule.Sandbox.create).toHaveBeenCalledTimes(1);
     expect(sandboxModule.Sandbox.create).toHaveBeenCalledWith(