|
| 1 | +#!/usr/bin/env npx tsx |
| 2 | +/** |
| 3 | + * Verify that the deployed ag-ui Railway runtime actually serves every topic |
| 4 | + * in the capability registry. |
| 5 | + * |
| 6 | + * Why this exists: `deploy-ag-ui.yml` ships with `railway up --detach`, which |
| 7 | + * returns as soon as the upload is accepted — long before Railway builds or |
| 8 | + * starts the image. A build that fails afterwards leaves the PREVIOUS image |
| 9 | + * running while the workflow reports success, so a topic can be missing from |
| 10 | + * production for months with every check green. That is exactly how |
| 11 | + * `/agent/subagents` went missing: three successful uploads carried the route, |
| 12 | + * but the live instance kept serving the pre-subagents route table. |
| 13 | + * |
| 14 | + * `/ok` cannot catch this — a stale image answers it happily. The only signal |
| 15 | + * that distinguishes image vintage is whether each topic's route is registered. |
| 16 | + * |
| 17 | + * Probing goes through the public Vercel proxy rather than Railway directly: |
| 18 | + * the FastAPI middleware requires X-Internal-Token, which lives on Railway and |
| 19 | + * Vercel but is deliberately not a CI secret. The proxy injects it. |
| 20 | + * |
| 21 | + * An empty POST body is a deliberate, token-free canary — the request model |
| 22 | + * rejects it before any graph or LLM call runs: |
| 23 | + * 422 → healthy (proxy routed, token accepted, topic registered) |
| 24 | + * 404 → topic missing from the deployed image, or no Vercel proxy route |
| 25 | + * 401 → AG_UI_INTERNAL_TOKEN mismatch between the proxy and Railway |
| 26 | + * |
| 27 | + * Usage: |
| 28 | + * npx tsx scripts/verify-ag-ui-runtime.ts |
| 29 | + * EXAMPLES_URL=https://examples.threadplane.ai npx tsx scripts/verify-ag-ui-runtime.ts |
| 30 | + */ |
| 31 | +import { pathToFileURL } from 'url'; |
| 32 | +import { capabilities } from '../apps/cockpit/scripts/capability-registry'; |
| 33 | + |
| 34 | +const EXAMPLES_URL = process.env['EXAMPLES_URL'] ?? 'https://examples.threadplane.ai'; |
| 35 | +const RAILWAY_URL = |
| 36 | + process.env['AG_UI_RAILWAY_URL'] ?? 'https://ag-ui-dev-production.up.railway.app'; |
| 37 | + |
| 38 | +/** How long to keep polling while Railway builds and rolls out the new image. */ |
| 39 | +const DEADLINE_MS = Number(process.env['AG_UI_VERIFY_TIMEOUT_MS'] ?? 15 * 60 * 1000); |
| 40 | +const POLL_INTERVAL_MS = Number(process.env['AG_UI_VERIFY_INTERVAL_MS'] ?? 20_000); |
| 41 | + |
| 42 | +export type TopicVerdict = |
| 43 | + | { ok: true; status: number } |
| 44 | + | { ok: false; status: number; reason: string }; |
| 45 | + |
| 46 | +/** |
| 47 | + * Classify an agent-endpoint response. Anything that is not a routing or auth |
| 48 | + * failure means the topic is registered and reachable — which is all this |
| 49 | + * check claims. |
| 50 | + */ |
| 51 | +export function classifyStatus(status: number): TopicVerdict { |
| 52 | + if (status === 404) { |
| 53 | + return { |
| 54 | + ok: false, |
| 55 | + status, |
| 56 | + reason: |
| 57 | + 'route not found — the deployed Railway image has no handler for this topic, or the Vercel proxy route is missing', |
| 58 | + }; |
| 59 | + } |
| 60 | + if (status === 401) { |
| 61 | + return { |
| 62 | + ok: false, |
| 63 | + status, |
| 64 | + reason: |
| 65 | + 'unauthorized — AG_UI_INTERNAL_TOKEN differs between the Vercel proxy and Railway', |
| 66 | + }; |
| 67 | + } |
| 68 | + if (status >= 500) { |
| 69 | + return { ok: false, status, reason: 'upstream error from the Railway runtime' }; |
| 70 | + } |
| 71 | + return { ok: true, status }; |
| 72 | +} |
| 73 | + |
| 74 | +export interface DeployedTopic { |
| 75 | + /** URL segment the examples site serves this capability under. */ |
| 76 | + product: 'ag-ui' | 'runtimes'; |
| 77 | + topic: string; |
| 78 | +} |
| 79 | + |
| 80 | +/** |
| 81 | + * Mirrors collectTopics() in scripts/generate-ag-ui-deployment-config.ts: the |
| 82 | + * 'ag-ui' and 'runtimes' products are both AG-UI-served FastAPI backends |
| 83 | + * aggregated into the single ag-ui-dev deployment, and a capability without a |
| 84 | + * pythonDir is hosted elsewhere (mastra is Node-hosted). Deriving the list the |
| 85 | + * same way the generator does keeps this check honest — it asserts exactly |
| 86 | + * what the deployment was generated to mount, so a new topic is covered with |
| 87 | + * no edit here. |
| 88 | + */ |
| 89 | +export function deployedTopics(): DeployedTopic[] { |
| 90 | + const topics = capabilities |
| 91 | + .filter( |
| 92 | + (c) => (c.product === 'ag-ui' || c.product === 'runtimes') && c.pythonDir |
| 93 | + ) |
| 94 | + .map((c) => ({ product: c.product as DeployedTopic['product'], topic: c.topic })) |
| 95 | + .sort((a, b) => a.topic.localeCompare(b.topic)); |
| 96 | + if (topics.length === 0) { |
| 97 | + throw new Error('No ag-ui topics with pythonDir found in capability registry'); |
| 98 | + } |
| 99 | + return topics; |
| 100 | +} |
| 101 | + |
| 102 | +/** Public path the SPA for this capability calls; both proxy to the same runtime. */ |
| 103 | +export function agentUrlFor({ product, topic }: DeployedTopic): string { |
| 104 | + return `/${product}/${topic}/agent`; |
| 105 | +} |
| 106 | + |
| 107 | +async function probeTopic(entry: DeployedTopic): Promise<TopicVerdict> { |
| 108 | + try { |
| 109 | + const res = await fetch(`${EXAMPLES_URL}${agentUrlFor(entry)}`, { |
| 110 | + method: 'POST', |
| 111 | + headers: { origin: EXAMPLES_URL, 'content-type': 'application/json' }, |
| 112 | + body: '{}', |
| 113 | + }); |
| 114 | + return classifyStatus(res.status); |
| 115 | + } catch (error) { |
| 116 | + const message = error instanceof Error ? error.message : String(error); |
| 117 | + return { ok: false, status: 0, reason: `request failed — ${message}` }; |
| 118 | + } |
| 119 | +} |
| 120 | + |
| 121 | +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); |
| 122 | + |
| 123 | +async function main(): Promise<void> { |
| 124 | + const topics = deployedTopics(); |
| 125 | + console.log( |
| 126 | + `Verifying ${topics.length} deployed topics via ${EXAMPLES_URL} (Railway: ${RAILWAY_URL})` |
| 127 | + ); |
| 128 | + |
| 129 | + const deadline = Date.now() + DEADLINE_MS; |
| 130 | + let pending: DeployedTopic[] = [...topics]; |
| 131 | + const verdicts = new Map<string, TopicVerdict>(); |
| 132 | + |
| 133 | + // Poll rather than probe once: `railway up --detach` returns before the new |
| 134 | + // image is live, so an immediate check would assert against the old one. |
| 135 | + for (;;) { |
| 136 | + const results = await Promise.all( |
| 137 | + pending.map(async (entry) => [entry, await probeTopic(entry)] as const) |
| 138 | + ); |
| 139 | + for (const [entry, verdict] of results) verdicts.set(entry.topic, verdict); |
| 140 | + pending = results.filter(([, v]) => !v.ok).map(([e]) => e); |
| 141 | + |
| 142 | + if (pending.length === 0) break; |
| 143 | + if (Date.now() >= deadline) break; |
| 144 | + |
| 145 | + const remainingS = Math.round((deadline - Date.now()) / 1000); |
| 146 | + console.log( |
| 147 | + `⏳ still unhealthy: ${pending.map((e) => e.topic).join(', ')} — retrying in ${POLL_INTERVAL_MS / 1000}s (${remainingS}s left)` |
| 148 | + ); |
| 149 | + await sleep(POLL_INTERVAL_MS); |
| 150 | + } |
| 151 | + |
| 152 | + const summary = topics.map((entry) => ({ |
| 153 | + topic: entry.topic, |
| 154 | + url: agentUrlFor(entry), |
| 155 | + verdict: verdicts.get(entry.topic) ?? { |
| 156 | + ok: false as const, |
| 157 | + status: 0, |
| 158 | + reason: 'never probed', |
| 159 | + }, |
| 160 | + })); |
| 161 | + |
| 162 | + for (const { topic, url, verdict } of summary) { |
| 163 | + if (verdict.ok) { |
| 164 | + console.log(`✅ ${topic}: mounted at ${url} (HTTP ${verdict.status})`); |
| 165 | + } else { |
| 166 | + console.error(`❌ ${topic}: ${url} → HTTP ${verdict.status} — ${verdict.reason}`); |
| 167 | + } |
| 168 | + } |
| 169 | + |
| 170 | + const failed = summary.filter(({ verdict }) => !verdict.ok); |
| 171 | + console.log(`\n${summary.length - failed.length} healthy, ${failed.length} failing`); |
| 172 | + if (failed.length > 0) { |
| 173 | + console.error( |
| 174 | + '\nThe deployed image does not serve every registered topic. Check the ' + |
| 175 | + 'Railway build log for the ag-ui-dev service — `railway up --detach` ' + |
| 176 | + 'reports success on upload, not on a successful build.' |
| 177 | + ); |
| 178 | + process.exit(1); |
| 179 | + } |
| 180 | +} |
| 181 | + |
| 182 | +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { |
| 183 | + main().catch((error) => { |
| 184 | + const message = error instanceof Error ? error.message : String(error); |
| 185 | + console.error(`❌ ag-ui runtime verification failed — ${message}`); |
| 186 | + process.exit(1); |
| 187 | + }); |
| 188 | +} |
0 commit comments