Skip to content

Commit 449dc0d

Browse files
committed
Add get-screenshot app tool to the map-server example
The map view already captures scaled PNG screenshots of the globe for model-context updates, but only pushed them opportunistically (and only when the host supports images in updateModelContext). This factors that capture into a shared helper and registers a get-screenshot view tool, so the model can request the current view on demand and see exactly what the user sees. Takes an optional maxDimension (default 1024px) to trade detail against image token cost. The e2e test follows the pdf-get-current-pdf pattern: it plays the host, posting tools/list and tools/call JSON-RPC into the app iframe from the sandbox frame.
1 parent 82221c0 commit 449dc0d

3 files changed

Lines changed: 248 additions & 26 deletions

File tree

‎examples/map-server/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,14 @@ Display the 3D globe zoomed to a bounding box.
9898

9999
Defaults to London if no coordinates provided.
100100

101+
### App tools
102+
103+
The map view registers tools on the view itself (via `app.registerTool`), for hosts that list and call view tools directly:
104+
105+
- `navigate-to` — fly the globe to a bounding box, with an optional label.
106+
- `get-current-view` — return the camera position and the visible bounding box (also as `structuredContent`).
107+
- `get-screenshot` — capture the current globe view as a PNG image in the tool result (optional `maxDimension`, default 1024px), so the model can see exactly what the user sees.
108+
101109
## Architecture
102110

103111
### Server (`server.ts`)

‎examples/map-server/src/mcp-app.ts‎

Lines changed: 86 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,32 @@ const CESIUM_VERSION = "1.123";
1717
const CESIUM_BASE_URL = `https://cesium.com/downloads/cesiumjs/releases/${CESIUM_VERSION}/Build/Cesium`;
1818

1919
const MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION = 768; // Max width/height for screenshots in pixels for updateModelContext
20+
const DEFAULT_SCREENSHOT_MAX_DIMENSION = 1024; // Default max width/height for the get-screenshot tool
21+
22+
/**
23+
* Capture the globe canvas as a base64 PNG, scaled down so neither side
24+
* exceeds maxDimension (image token cost scales with dimensions).
25+
*/
26+
function captureScreenshotBase64(
27+
sourceCanvas: HTMLCanvasElement,
28+
maxDimension: number,
29+
): { data: string; width: number; height: number } | null {
30+
const scale = Math.min(
31+
1,
32+
maxDimension / Math.max(sourceCanvas.width, sourceCanvas.height),
33+
);
34+
const targetWidth = Math.round(sourceCanvas.width * scale);
35+
const targetHeight = Math.round(sourceCanvas.height * scale);
36+
const tempCanvas = document.createElement("canvas");
37+
tempCanvas.width = targetWidth;
38+
tempCanvas.height = targetHeight;
39+
const ctx = tempCanvas.getContext("2d");
40+
if (!ctx) return null;
41+
ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight);
42+
const base64Data = tempCanvas.toDataURL("image/png").split(",")[1];
43+
if (!base64Data) return null;
44+
return { data: base64Data, width: targetWidth, height: targetHeight };
45+
}
2046

2147
/**
2248
* Dynamically load CesiumJS from CDN
@@ -300,33 +326,19 @@ function scheduleLocationUpdate(cesiumViewer: any): void {
300326
if (app.getHostCapabilities()?.updateModelContext?.image) {
301327
try {
302328
// Scale down to reduce token usage (tokens depend on dimensions)
303-
const sourceCanvas = cesiumViewer.canvas;
304-
const scale = Math.min(
305-
1,
306-
MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION /
307-
Math.max(sourceCanvas.width, sourceCanvas.height),
329+
const shot = captureScreenshotBase64(
330+
cesiumViewer.canvas,
331+
MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION,
308332
);
309-
const targetWidth = Math.round(sourceCanvas.width * scale);
310-
const targetHeight = Math.round(sourceCanvas.height * scale);
311-
312-
const tempCanvas = document.createElement("canvas");
313-
tempCanvas.width = targetWidth;
314-
tempCanvas.height = targetHeight;
315-
const ctx = tempCanvas.getContext("2d");
316-
if (ctx) {
317-
ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight);
318-
const dataUrl = tempCanvas.toDataURL("image/png");
319-
const base64Data = dataUrl.split(",")[1];
320-
if (base64Data) {
321-
content.push({
322-
type: "image",
323-
data: base64Data,
324-
mimeType: "image/png",
325-
});
326-
log.info(
327-
`Added screenshot to model context (${targetWidth}x${targetHeight})`,
328-
);
329-
}
333+
if (shot) {
334+
content.push({
335+
type: "image",
336+
data: shot.data,
337+
mimeType: "image/png",
338+
});
339+
log.info(
340+
`Added screenshot to model context (${shot.width}x${shot.height})`,
341+
);
330342
}
331343
} catch (err) {
332344
log.warn("Failed to capture screenshot:", err);
@@ -977,6 +989,54 @@ app.registerTool(
977989
},
978990
);
979991

992+
app.registerTool(
993+
"get-screenshot",
994+
{
995+
title: "Get Screenshot",
996+
description:
997+
"Capture the current globe view as a PNG image for visual analysis",
998+
inputSchema: z.object({
999+
maxDimension: z
1000+
.number()
1001+
.int()
1002+
.min(64)
1003+
.max(2048)
1004+
.optional()
1005+
.describe(
1006+
"Max width/height in pixels (default 1024). Larger images show more detail but cost more tokens.",
1007+
),
1008+
}),
1009+
},
1010+
async (args) => {
1011+
if (!viewer) {
1012+
return {
1013+
content: [
1014+
{ type: "text" as const, text: "Error: Viewer not initialized" },
1015+
],
1016+
isError: true,
1017+
};
1018+
}
1019+
1020+
const shot = captureScreenshotBase64(
1021+
viewer.canvas,
1022+
args.maxDimension ?? DEFAULT_SCREENSHOT_MAX_DIMENSION,
1023+
);
1024+
if (!shot) {
1025+
return {
1026+
content: [
1027+
{ type: "text" as const, text: "Error: screenshot capture failed" },
1028+
],
1029+
isError: true,
1030+
};
1031+
}
1032+
return {
1033+
content: [
1034+
{ type: "image" as const, data: shot.data, mimeType: "image/png" },
1035+
],
1036+
};
1037+
},
1038+
);
1039+
9801040
// Handle tool result - extract viewUUID and restore persisted view if available
9811041
app.ontoolresult = async (result) => {
9821042
viewUUID = result._meta?.viewUUID ? String(result._meta.viewUUID) : undefined;
Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/**
2+
* Map Server — get-screenshot app tool
3+
*
4+
* The map view registers tools on the view itself (App.registerTool). A host
5+
* calls them by sending `tools/call` JSON-RPC requests to the app iframe over
6+
* its PostMessageTransport, which accepts messages whose source is
7+
* window.parent. The basic-host test UI only drives server tools, so this
8+
* test plays the host's role directly: it posts the JSON-RPC request from the
9+
* sandbox frame (the app iframe's parent) and reads the response there.
10+
*/
11+
12+
import { test, expect, type Page, type Frame } from "@playwright/test";
13+
14+
/** Wait for the MCP App to load inside nested iframes. */
15+
async function waitForAppLoad(page: Page) {
16+
const outerFrame = page.frameLocator("iframe").first();
17+
await expect(outerFrame.locator("iframe")).toBeVisible({ timeout: 30000 });
18+
}
19+
20+
/** Get the app frame locator (nested: sandbox > app) */
21+
function getAppFrame(page: Page) {
22+
return page.frameLocator("iframe").first().frameLocator("iframe").first();
23+
}
24+
25+
/** Load the map server and call show-map with its default bounding box. */
26+
async function loadMapServer(page: Page) {
27+
await page.goto("/?theme=hide");
28+
// The host fetches the server list once per page load, and the example
29+
// servers rebuild on examples:start, so a server can come up after the
30+
// first load — reload until it appears instead of waiting on a stale list.
31+
await expect(async () => {
32+
await page.reload();
33+
const serverSelect = page.locator("select").first();
34+
await expect(serverSelect).toBeEnabled({ timeout: 30000 });
35+
expect(await serverSelect.locator("option").allTextContents()).toContain(
36+
"CesiumJS Map Server",
37+
);
38+
}).toPass({ timeout: 90000, intervals: [2000] });
39+
await page
40+
.locator("select")
41+
.first()
42+
.selectOption({ label: "CesiumJS Map Server" });
43+
await page.locator("select").nth(1).selectOption("show-map");
44+
await page.click('button:has-text("Call Tool")');
45+
await waitForAppLoad(page);
46+
}
47+
48+
/** Wait for the Cesium canvas to render and give tiles time to load. */
49+
async function waitForGlobeCanvas(page: Page) {
50+
const appFrame = getAppFrame(page);
51+
await expect(appFrame.locator("canvas").first()).toBeVisible({
52+
timeout: 60000,
53+
});
54+
// CesiumJS needs time for tiles to load (same budget as servers.spec.ts)
55+
await page.waitForTimeout(15000);
56+
}
57+
58+
/** Resolve the sandbox frame (the app iframe's parent). */
59+
function getSandboxFrame(page: Page): Frame {
60+
const appFrame = page
61+
.frames()
62+
.find((f) => f.parentFrame() && f.parentFrame()!.parentFrame());
63+
if (!appFrame) throw new Error("App frame not found");
64+
return appFrame.parentFrame()!;
65+
}
66+
67+
/**
68+
* Send one JSON-RPC request into the app iframe from the sandbox frame and
69+
* await the matching response.
70+
*/
71+
function sendJsonRpcToApp(
72+
sandboxFrame: Frame,
73+
request: { id: number; method: string; params?: unknown },
74+
timeoutMs = 60_000,
75+
): Promise<unknown> {
76+
return sandboxFrame.evaluate(
77+
async ({ id, method, params, timeoutMs }) => {
78+
const iframe = document.querySelector("iframe") as HTMLIFrameElement;
79+
return await new Promise<unknown>((resolve, reject) => {
80+
const timer = setTimeout(
81+
() => reject(new Error(`${method} timed out`)),
82+
timeoutMs,
83+
);
84+
const onMsg = (e: MessageEvent) => {
85+
// Only accept a response (not an echoed request) to our id, from
86+
// the app iframe itself.
87+
if (
88+
e.source === iframe.contentWindow &&
89+
e.data?.jsonrpc === "2.0" &&
90+
e.data.id === id &&
91+
("result" in e.data || "error" in e.data)
92+
) {
93+
clearTimeout(timer);
94+
window.removeEventListener("message", onMsg);
95+
resolve(e.data);
96+
}
97+
};
98+
window.addEventListener("message", onMsg);
99+
iframe.contentWindow!.postMessage(
100+
{ jsonrpc: "2.0", id, method, params },
101+
"*",
102+
);
103+
});
104+
},
105+
{ ...request, timeoutMs },
106+
);
107+
}
108+
109+
test.describe("Map Server - get-screenshot", () => {
110+
test("is listed and returns the current view as a PNG image", async ({
111+
page,
112+
}) => {
113+
// Server-list reload loop + CDN Cesium + tile loading add up.
114+
test.setTimeout(240_000);
115+
await loadMapServer(page);
116+
await waitForGlobeCanvas(page);
117+
118+
const sandboxFrame = getSandboxFrame(page);
119+
120+
const listResponse = (await sendJsonRpcToApp(sandboxFrame, {
121+
id: 9201,
122+
method: "tools/list",
123+
params: {},
124+
})) as { result: { tools: { name: string }[] } };
125+
expect(listResponse.result.tools.map((t) => t.name)).toContain(
126+
"get-screenshot",
127+
);
128+
129+
const callResponse = (await sendJsonRpcToApp(sandboxFrame, {
130+
id: 9202,
131+
method: "tools/call",
132+
params: { name: "get-screenshot", arguments: { maxDimension: 512 } },
133+
})) as {
134+
result: {
135+
isError?: boolean;
136+
content: {
137+
type: string;
138+
text?: string;
139+
data?: string;
140+
mimeType?: string;
141+
}[];
142+
};
143+
};
144+
145+
const result = callResponse.result;
146+
expect(result.isError).toBeFalsy();
147+
const image = result.content.find((c) => c.type === "image");
148+
expect(image).toBeTruthy();
149+
expect(image!.mimeType).toBe("image/png");
150+
// A real PNG: base64 of the \x89PNG magic is "iVBOR"
151+
expect(image!.data!.slice(0, 5)).toBe("iVBOR");
152+
expect(image!.data!.length).toBeGreaterThan(1000);
153+
});
154+
});

0 commit comments

Comments
 (0)