diff --git a/.github/workflows/sync-surface-verification.yml b/.github/workflows/sync-surface-verification.yml index ee02b25c09..577ba1a097 100644 --- a/.github/workflows/sync-surface-verification.yml +++ b/.github/workflows/sync-surface-verification.yml @@ -13,7 +13,7 @@ name: Sync surface verification # them continuously. The evidence existed; nothing carried it. # # The snapshot is COMMITTED rather than fetched during the build because -# tests/artifacts.test.ts asserts the artifact build is byte-identical across +# tests/artifacts-build-determinism.test.ts asserts the artifact build is byte-identical across # rebuilds, and a network call inside the build would end that -- as well as # making every build depend on the API being up. Same posture as # registry/verification/promotions.json. diff --git a/scripts/sync-surface-verification.ts b/scripts/sync-surface-verification.ts index 43362deac2..6d29af40ba 100644 --- a/scripts/sync-surface-verification.ts +++ b/scripts/sync-surface-verification.ts @@ -7,7 +7,7 @@ // the evidence comes from the health prober rather than verify-candidates.ts, // and for the promotion bar itself. // -// COMMITTED, not fetched at build time. tests/artifacts.test.ts asserts the +// COMMITTED, not fetched at build time. tests/artifacts-build-determinism.test.ts asserts the // artifact build is byte-identical across rebuilds; a network call inside the // build would end that, and would also make every build depend on the API being // up. Same posture as registry/verification/promotions.json. diff --git a/tests/artifacts-build-determinism.test.ts b/tests/artifacts-build-determinism.test.ts new file mode 100644 index 0000000000..d4d63400ac --- /dev/null +++ b/tests/artifacts-build-determinism.test.ts @@ -0,0 +1,89 @@ +// Split out of tests/artifacts.test.ts (#8937 follow-up). Each test here runs a +// full scripts/build-artifacts.ts, which is 7-25s of CI on its own; six of them +// sharing one file made artifacts.test.ts 118.8s of a 144.3s pass and the floor +// for the whole shared-registry run, because vitest parallelizes across FILES +// and never within one. One file per build lets them run on separate workers. +// +// The per-file sandbox clone is ~0.6s against a ~20s build, so the split pays +// for itself many times over. +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { test } from "vitest"; + +import { createArtifactBuildHarness } from "./helpers/artifact-build-harness.ts"; +import type { Row } from "./row-type.ts"; + +const harness = createArtifactBuildHarness("artifacts-build-determinism"); + +// #510 refactor invariant: the artifact build is deterministic, so two +// consecutive builds (epoch timestamp, no METAGRAPH_BUILD_TIMESTAMP) must emit a +// byte-identical R2 staging tree. This is the regression guard that lets the +// build-artifacts/lib decomposition stay safe — any future code-motion that +// silently reorders keys, changes a number, or drops an artifact flips this hash. +// It deliberately compares the whole staging tree (not a hardcoded golden), so it +// never needs touching when the committed source data legitimately refreshes. +function digestArtifactTree(root: string) { + const hash = createHash("sha256"); + for (const file of harness + .walkFiles(root) + .filter((file) => path.basename(file) !== ".DS_Store") // OS noise, not an artifact + .sort()) { + hash.update(path.relative(root, file)); + hash.update("\0"); + hash.update(readFileSync(file)); + hash.update("\0"); + } + return hash.digest("hex"); +} + +test("artifact build is deterministic (byte-identical across rebuilds)", () => { + const supportArtifacts = harness.snapshotSupportArtifacts(); + const buildEnv: Row = { + ...harness.env, + METAGRAPH_PRESERVE_PROBE_HEALTH: "1", + }; + delete buildEnv.METAGRAPH_BUILD_TIMESTAMP; // force the reproducible epoch + const runBuild = () => + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: buildEnv as unknown as NodeJS.ProcessEnv, + stdio: "pipe", + }); + try { + runBuild(); + const firstDigest = digestArtifactTree(harness.r2StagingRoot); + + // The build must actually produce the artifacts whose derivation was + // extracted to scripts/lib/ — a broken import would yield empty/missing + // output, which this asserts before the cheaper hash comparison. + for (const relativePath of [ + "endpoints.json", + "rpc-endpoints.json", + "economics.json", + "endpoint-pools.json", + "endpoint-incidents.json", + ]) { + const artifact = harness.readArtifact(relativePath); + assert.ok( + artifact && typeof artifact === "object", + `${relativePath} should build to a non-empty object`, + ); + } + + runBuild(); + const secondDigest = digestArtifactTree(harness.r2StagingRoot); + + assert.equal( + secondDigest, + firstDigest, + "two consecutive builds must emit a byte-identical R2 staging tree", + ); + } finally { + runBuild(); + harness.restoreSupportArtifacts(supportArtifacts); + } +}, 120_000); diff --git a/tests/artifacts-build-health.test.ts b/tests/artifacts-build-health.test.ts new file mode 100644 index 0000000000..40e21a72c6 --- /dev/null +++ b/tests/artifacts-build-health.test.ts @@ -0,0 +1,109 @@ +// Split out of tests/artifacts.test.ts (#8937 follow-up). Each test here runs a +// full scripts/build-artifacts.ts, which is 7-25s of CI on its own; six of them +// sharing one file made artifacts.test.ts 118.8s of a 144.3s pass and the floor +// for the whole shared-registry run, because vitest parallelizes across FILES +// and never within one. One file per build lets them run on separate workers. +// +// The per-file sandbox clone is ~0.6s against a ~20s build, so the split pays +// for itself many times over. +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { test } from "vitest"; + +import { createArtifactBuildHarness } from "./helpers/artifact-build-harness.ts"; +import type { Row } from "./row-type.ts"; + +const harness = createArtifactBuildHarness("artifacts-build-health"); + +test("artifact build does not preserve forged endpoint index health", () => { + const endpointsPath = harness.artifactFilePath("endpoints.json"); + // Sandbox-rooted, for the same reason the harness roots its support-artifact + // paths. Relative, this pointed at the real repo, which also made the setup a + // no-op against its own purpose: the build reads the SANDBOX's .cache, so + // clearing the real one never removed the health cache the rebuild consults. + const cachePath = path.join( + harness.root, + ".cache/metagraphed/health/latest.json", + ); + const original = readFileSync(endpointsPath, "utf8"); + const originalCache = existsSync(cachePath) + ? readFileSync(cachePath, "utf8") + : null; + const supportArtifacts = harness.snapshotSupportArtifacts(); + rmSync(cachePath, { force: true }); + const tampered = JSON.parse(original); + const target = tampered.endpoints.find( + (endpoint: Row) => endpoint.public_safe === true, + ); + assert(target, "expected a public-safe endpoint row to tamper"); + + target.health_source = "probe-derived"; + target.monitoring_status = "monitored"; + target.status = "ok"; + target.classification = "live"; + target.last_checked = "2999-01-01T00:00:00.000Z"; + target.last_ok = "2999-01-01T00:00:00.000Z"; + target.observed_at = "2999-01-01T00:00:00.000Z"; + target.latency_ms = 7; + target.latest_block = 4242424242; + target.archive_support = true; + + try { + writeFileSync(endpointsPath, `${JSON.stringify(tampered, null, 2)}\n`); + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: { ...harness.env, METAGRAPH_PRESERVE_PROBE_HEALTH: "1" }, + stdio: "pipe", + }); + + const rebuilt = JSON.parse(readFileSync(endpointsPath, "utf8")); + const rebuiltTarget = rebuilt.endpoints.find( + (endpoint: Row) => endpoint.surface_id === target.surface_id, + ); + assert.equal(rebuiltTarget.status, "unknown"); + assert.equal(rebuiltTarget.classification, "unknown"); + assert.equal(rebuiltTarget.last_checked, null); + assert.equal(rebuiltTarget.latency_ms, null); + assert.equal(rebuiltTarget.latest_block, null); + assert.equal(rebuiltTarget.archive_support, null); + assert.equal(rebuiltTarget.health_source, "missing-probe"); + } finally { + writeFileSync(endpointsPath, original); + if (originalCache === null) { + rmSync(cachePath, { force: true }); + } else { + writeFileSync(cachePath, originalCache); + } + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: { + ...harness.env, + METAGRAPH_PRESERVE_PROBE_HEALTH: "1", + }, + stdio: "pipe", + }); + execFileSync(process.execPath, ["scripts/generate-types.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + execFileSync(process.execPath, ["scripts/generate-client.ts", "--write"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + execFileSync(process.execPath, ["scripts/r2-manifest.ts", "--write"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + harness.restoreSupportArtifacts(supportArtifacts); + } +}, 120_000); diff --git a/tests/artifacts-build-schema-index.test.ts b/tests/artifacts-build-schema-index.test.ts new file mode 100644 index 0000000000..839309547b --- /dev/null +++ b/tests/artifacts-build-schema-index.test.ts @@ -0,0 +1,110 @@ +// Split out of tests/artifacts.test.ts (#8937 follow-up). Each test here runs a +// full scripts/build-artifacts.ts, which is 7-25s of CI on its own; six of them +// sharing one file made artifacts.test.ts 118.8s of a 144.3s pass and the floor +// for the whole shared-registry run, because vitest parallelizes across FILES +// and never within one. One file per build lets them run on separate workers. +// +// The per-file sandbox clone is ~0.6s against a ~20s build, so the split pays +// for itself many times over. +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { + cpSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { test } from "vitest"; + +import { createArtifactBuildHarness } from "./helpers/artifact-build-harness.ts"; +import type { Row } from "./row-type.ts"; + +const harness = createArtifactBuildHarness("artifacts-build-schema-index"); + +test("artifact build preserves committed schema index without R2 schema details", () => { + const schemaIndexPath = harness.artifactFilePath("schemas/index.json"); + const originalSchemaIndex = readFileSync(schemaIndexPath, "utf8"); + const originalSchemaIndexJson = JSON.parse(originalSchemaIndex); + const supportArtifacts = harness.snapshotSupportArtifacts(); + const backupDir = mkdtempSync(`${tmpdir()}/metagraphed-schema-r2-`); + const stagingBackup = `${backupDir}/metagraph-r2`; + const hadStagingRoot = existsSync(harness.r2StagingRoot); + if (hadStagingRoot) { + cpSync(harness.r2StagingRoot, stagingBackup, { recursive: true }); + } + + assert.equal(originalSchemaIndexJson.source, "openapi-snapshot"); + assert.equal(originalSchemaIndexJson.schemas.length > 0, true); + + try { + rmSync(harness.r2StagingRoot, { recursive: true, force: true }); + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + + const rebuiltSchemaIndex = readFileSync(schemaIndexPath, "utf8"); + assert.deepEqual(JSON.parse(rebuiltSchemaIndex), originalSchemaIndexJson); + } finally { + writeFileSync(schemaIndexPath, originalSchemaIndex); + rmSync(harness.r2StagingRoot, { recursive: true, force: true }); + if (hadStagingRoot) { + cpSync(stagingBackup, harness.r2StagingRoot, { recursive: true }); + } + harness.restoreSupportArtifacts(supportArtifacts); + rmSync(backupDir, { recursive: true, force: true }); + } +}, 120_000); + +test("artifact build accepts an OpenAPI-vendor JSON content-type for a captured schema entry", () => { + const schemaIndexPath = harness.artifactFilePath("schemas/index.json"); + const originalSchemaIndex = readFileSync(schemaIndexPath, "utf8"); + const supportArtifacts = harness.snapshotSupportArtifacts(); + const schemaIndex = JSON.parse(originalSchemaIndex); + const indexTarget = schemaIndex.schemas?.find( + (schema: Row) => schema.status === "captured", + ); + assert(indexTarget, "expected a captured schema index entry to retype"); + + // A real subnet (SN-71 Leadpoet) serves its OpenAPI document as + // application/vnd.oai.openapi+json rather than plain application/json -- a + // spec-valid, OAI-registered media type. schemaIndexEntryMatchesSurface used + // to require an exact "application/json" match, so this one entry failed the + // reconciler's forgery/staleness guard and wholesale-discarded the entire + // committed index down to an empty placeholder (metagraphed#6411). + indexTarget.content_type = "application/vnd.oai.openapi+json; charset=utf-8"; + + try { + writeFileSync(schemaIndexPath, `${JSON.stringify(schemaIndex, null, 2)}\n`); + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + + const rebuiltSchemaIndex = JSON.parse( + readFileSync(schemaIndexPath, "utf8"), + ); + assert.equal(rebuiltSchemaIndex.source, "openapi-snapshot"); + const rebuiltTarget = rebuiltSchemaIndex.schemas.find( + (schema: Row) => schema.surface_id === indexTarget.surface_id, + ); + assert.equal(rebuiltTarget?.content_type, indexTarget.content_type); + assert.equal(rebuiltTarget?.hash, indexTarget.hash); + } finally { + writeFileSync(schemaIndexPath, originalSchemaIndex); + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + harness.restoreSupportArtifacts(supportArtifacts); + } +}, 120_000); diff --git a/tests/artifacts-build-schema.test.ts b/tests/artifacts-build-schema.test.ts new file mode 100644 index 0000000000..911acf8f8c --- /dev/null +++ b/tests/artifacts-build-schema.test.ts @@ -0,0 +1,121 @@ +// Split out of tests/artifacts.test.ts (#8937 follow-up). Each test here runs a +// full scripts/build-artifacts.ts, which is 7-25s of CI on its own; six of them +// sharing one file made artifacts.test.ts 118.8s of a 144.3s pass and the floor +// for the whole shared-registry run, because vitest parallelizes across FILES +// and never within one. One file per build lets them run on separate workers. +// +// The per-file sandbox clone is ~0.6s against a ~20s build, so the split pays +// for itself many times over. +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { test } from "vitest"; + +import { createArtifactBuildHarness } from "./helpers/artifact-build-harness.ts"; +import type { Row } from "./row-type.ts"; + +const harness = createArtifactBuildHarness("artifacts-build-schema"); + +test("artifact build does not preserve forged schema snapshot metadata", () => { + const schemaDriftPath = harness.artifactFilePath("schema-drift.json"); + const schemaIndexPath = harness.artifactFilePath("schemas/index.json"); + const originalSchemaDrift = existsSync(schemaDriftPath) + ? readFileSync(schemaDriftPath, "utf8") + : null; + const originalSchemaIndex = readFileSync(schemaIndexPath, "utf8"); + const supportArtifacts = harness.snapshotSupportArtifacts(); + const schemaDrift = originalSchemaDrift + ? JSON.parse(originalSchemaDrift) + : null; + const schemaIndex = JSON.parse(originalSchemaIndex); + const driftTarget = schemaDrift?.surfaces?.[0]; + const indexTarget = + schemaIndex.schemas?.find( + (schema: Row) => schema.surface_id === driftTarget?.surface_id, + ) || + schemaIndex.schemas?.find((schema: Row) => schema.status === "captured"); + assert(indexTarget, "expected a schema index entry to tamper"); + + const forgedMarker = "AUTOVALIDATOR_FORGED_METADATA_SHOULD_NOT_SURVIVE_BUILD"; + if (driftTarget) { + driftTarget.netuid = 999999; + driftTarget.subnet_slug = forgedMarker; + driftTarget.url = "https://attacker.invalid/openapi"; + driftTarget.schema_url = "https://attacker.invalid/openapi.json"; + driftTarget.hash = "forged-hash"; + } + indexTarget.netuid = 999999; + indexTarget.subnet_slug = forgedMarker; + indexTarget.url = "https://attacker.invalid/openapi"; + indexTarget.schema_url = "https://attacker.invalid/openapi.json"; + indexTarget.hash = "forged-hash"; + indexTarget.path = "/metagraph/schemas/forged-by-autovalidator.json"; + indexTarget.snapshot = { + ...indexTarget.snapshot, + netuid: 999999, + subnet_slug: forgedMarker, + surface_url: "https://attacker.invalid/openapi", + schema_url: "https://attacker.invalid/openapi.json", + hash: "forged-hash", + title: forgedMarker, + }; + + try { + if (schemaDrift) { + writeFileSync( + schemaDriftPath, + `${JSON.stringify(schemaDrift, null, 2)}\n`, + ); + } + writeFileSync(schemaIndexPath, `${JSON.stringify(schemaIndex, null, 2)}\n`); + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + + const rebuiltSchemaDrift = existsSync(schemaDriftPath) + ? readFileSync(schemaDriftPath, "utf8") + : ""; + const rebuiltSchemaIndex = readFileSync(schemaIndexPath, "utf8"); + assert.equal(rebuiltSchemaDrift.includes(forgedMarker), false); + assert.equal(rebuiltSchemaIndex.includes(forgedMarker), false); + if (rebuiltSchemaDrift) { + assert.equal(JSON.parse(rebuiltSchemaDrift).source, "artifact-build"); + } + assert.equal(JSON.parse(rebuiltSchemaIndex).source, "artifact-build"); + } finally { + if (originalSchemaDrift) { + writeFileSync(schemaDriftPath, originalSchemaDrift); + } else { + rmSync(schemaDriftPath, { force: true }); + } + writeFileSync(schemaIndexPath, originalSchemaIndex); + execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + execFileSync(process.execPath, ["scripts/generate-types.ts"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + execFileSync(process.execPath, ["scripts/generate-client.ts", "--write"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + execFileSync(process.execPath, ["scripts/r2-manifest.ts", "--write"], { + cwd: harness.scriptCwd, + encoding: "utf8", + env: harness.env, + stdio: "pipe", + }); + harness.restoreSupportArtifacts(supportArtifacts); + } +}, 120_000); diff --git a/tests/artifacts.test.ts b/tests/artifacts.test.ts index 3d4cb10b93..98931a15e2 100644 --- a/tests/artifacts.test.ts +++ b/tests/artifacts.test.ts @@ -1,6 +1,5 @@ import assert from "node:assert/strict"; import { execFile, execFileSync, spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; import { cpSync, existsSync, @@ -64,16 +63,6 @@ const artifactDirectoryPath = (relativePath: string) => const publicMetagraphRoot = sandbox.publicMetagraphRoot; const r2StagingRoot = sandbox.r2StagingRoot; -// Sandbox-rooted, NOT relative. A bare relative path resolves against -// process.cwd() — the real repo — so restoreSupportArtifacts would truncate and -// rewrite the committed public/metagraph/r2-manifest.json while the rest of the -// suite runs. writeFileSync is not atomic, so any of the eight other test files -// that read that manifest could observe it empty or half-written. Rooting it in -// the sandbox is the point of #8937: the build writes only its own tree. -const SUPPORT_ARTIFACT_PATHS = [ - path.join(sandbox.root, "public/metagraph/r2-manifest.json"), -]; - function runNode(script: string) { execFileSync(process.execPath, [script], { cwd: sandbox.scriptCwd, @@ -400,355 +389,6 @@ test("registry validation rejects tampered per-subnet artifacts", () => { ); }); -test("artifact build does not preserve forged endpoint index health", () => { - const endpointsPath = artifactFilePath("endpoints.json"); - // Sandbox-rooted for the same reason as SUPPORT_ARTIFACT_PATHS. Relative, it - // pointed at the real repo, which also made the setup a no-op against its own - // purpose: the build reads the SANDBOX's .cache, so clearing the real one - // never removed the health cache the rebuild would actually consult. - const cachePath = path.join( - sandbox.root, - ".cache/metagraphed/health/latest.json", - ); - const original = readFileSync(endpointsPath, "utf8"); - const originalCache = existsSync(cachePath) - ? readFileSync(cachePath, "utf8") - : null; - const supportArtifacts = snapshotSupportArtifacts(); - rmSync(cachePath, { force: true }); - const tampered = JSON.parse(original); - const target = tampered.endpoints.find( - (endpoint: Row) => endpoint.public_safe === true, - ); - assert(target, "expected a public-safe endpoint row to tamper"); - - target.health_source = "probe-derived"; - target.monitoring_status = "monitored"; - target.status = "ok"; - target.classification = "live"; - target.last_checked = "2999-01-01T00:00:00.000Z"; - target.last_ok = "2999-01-01T00:00:00.000Z"; - target.observed_at = "2999-01-01T00:00:00.000Z"; - target.latency_ms = 7; - target.latest_block = 4242424242; - target.archive_support = true; - - try { - writeFileSync(endpointsPath, `${JSON.stringify(tampered, null, 2)}\n`); - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: { ...sandbox.env, METAGRAPH_PRESERVE_PROBE_HEALTH: "1" }, - stdio: "pipe", - }); - - const rebuilt = JSON.parse(readFileSync(endpointsPath, "utf8")); - const rebuiltTarget = rebuilt.endpoints.find( - (endpoint: Row) => endpoint.surface_id === target.surface_id, - ); - assert.equal(rebuiltTarget.status, "unknown"); - assert.equal(rebuiltTarget.classification, "unknown"); - assert.equal(rebuiltTarget.last_checked, null); - assert.equal(rebuiltTarget.latency_ms, null); - assert.equal(rebuiltTarget.latest_block, null); - assert.equal(rebuiltTarget.archive_support, null); - assert.equal(rebuiltTarget.health_source, "missing-probe"); - } finally { - writeFileSync(endpointsPath, original); - if (originalCache === null) { - rmSync(cachePath, { force: true }); - } else { - writeFileSync(cachePath, originalCache); - } - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: { - ...sandbox.env, - METAGRAPH_PRESERVE_PROBE_HEALTH: "1", - }, - stdio: "pipe", - }); - execFileSync(process.execPath, ["scripts/generate-types.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - execFileSync(process.execPath, ["scripts/generate-client.ts", "--write"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - execFileSync(process.execPath, ["scripts/r2-manifest.ts", "--write"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - restoreSupportArtifacts(supportArtifacts); - } -}, 30_000); - -test("artifact build does not preserve forged schema snapshot metadata", () => { - const schemaDriftPath = artifactFilePath("schema-drift.json"); - const schemaIndexPath = artifactFilePath("schemas/index.json"); - const originalSchemaDrift = existsSync(schemaDriftPath) - ? readFileSync(schemaDriftPath, "utf8") - : null; - const originalSchemaIndex = readFileSync(schemaIndexPath, "utf8"); - const supportArtifacts = snapshotSupportArtifacts(); - const schemaDrift = originalSchemaDrift - ? JSON.parse(originalSchemaDrift) - : null; - const schemaIndex = JSON.parse(originalSchemaIndex); - const driftTarget = schemaDrift?.surfaces?.[0]; - const indexTarget = - schemaIndex.schemas?.find( - (schema: Row) => schema.surface_id === driftTarget?.surface_id, - ) || - schemaIndex.schemas?.find((schema: Row) => schema.status === "captured"); - assert(indexTarget, "expected a schema index entry to tamper"); - - const forgedMarker = "AUTOVALIDATOR_FORGED_METADATA_SHOULD_NOT_SURVIVE_BUILD"; - if (driftTarget) { - driftTarget.netuid = 999999; - driftTarget.subnet_slug = forgedMarker; - driftTarget.url = "https://attacker.invalid/openapi"; - driftTarget.schema_url = "https://attacker.invalid/openapi.json"; - driftTarget.hash = "forged-hash"; - } - indexTarget.netuid = 999999; - indexTarget.subnet_slug = forgedMarker; - indexTarget.url = "https://attacker.invalid/openapi"; - indexTarget.schema_url = "https://attacker.invalid/openapi.json"; - indexTarget.hash = "forged-hash"; - indexTarget.path = "/metagraph/schemas/forged-by-autovalidator.json"; - indexTarget.snapshot = { - ...indexTarget.snapshot, - netuid: 999999, - subnet_slug: forgedMarker, - surface_url: "https://attacker.invalid/openapi", - schema_url: "https://attacker.invalid/openapi.json", - hash: "forged-hash", - title: forgedMarker, - }; - - try { - if (schemaDrift) { - writeFileSync( - schemaDriftPath, - `${JSON.stringify(schemaDrift, null, 2)}\n`, - ); - } - writeFileSync(schemaIndexPath, `${JSON.stringify(schemaIndex, null, 2)}\n`); - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - - const rebuiltSchemaDrift = existsSync(schemaDriftPath) - ? readFileSync(schemaDriftPath, "utf8") - : ""; - const rebuiltSchemaIndex = readFileSync(schemaIndexPath, "utf8"); - assert.equal(rebuiltSchemaDrift.includes(forgedMarker), false); - assert.equal(rebuiltSchemaIndex.includes(forgedMarker), false); - if (rebuiltSchemaDrift) { - assert.equal(JSON.parse(rebuiltSchemaDrift).source, "artifact-build"); - } - assert.equal(JSON.parse(rebuiltSchemaIndex).source, "artifact-build"); - } finally { - if (originalSchemaDrift) { - writeFileSync(schemaDriftPath, originalSchemaDrift); - } else { - rmSync(schemaDriftPath, { force: true }); - } - writeFileSync(schemaIndexPath, originalSchemaIndex); - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - execFileSync(process.execPath, ["scripts/generate-types.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - execFileSync(process.execPath, ["scripts/generate-client.ts", "--write"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - execFileSync(process.execPath, ["scripts/r2-manifest.ts", "--write"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - restoreSupportArtifacts(supportArtifacts); - } -}, 30_000); - -// #510 refactor invariant: the artifact build is deterministic, so two -// consecutive builds (epoch timestamp, no METAGRAPH_BUILD_TIMESTAMP) must emit a -// byte-identical R2 staging tree. This is the regression guard that lets the -// build-artifacts/lib decomposition stay safe — any future code-motion that -// silently reorders keys, changes a number, or drops an artifact flips this hash. -// It deliberately compares the whole staging tree (not a hardcoded golden), so it -// never needs touching when the committed source data legitimately refreshes. -function digestArtifactTree(root: string) { - const hash = createHash("sha256"); - for (const file of walkFilesRecursive(root) - .filter((file) => path.basename(file) !== ".DS_Store") // OS noise, not an artifact - .sort()) { - hash.update(path.relative(root, file)); - hash.update("\0"); - hash.update(readFileSync(file)); - hash.update("\0"); - } - return hash.digest("hex"); -} - -test("artifact build is deterministic (byte-identical across rebuilds)", () => { - const supportArtifacts = snapshotSupportArtifacts(); - const buildEnv: Row = { - ...sandbox.env, - METAGRAPH_PRESERVE_PROBE_HEALTH: "1", - }; - delete buildEnv.METAGRAPH_BUILD_TIMESTAMP; // force the reproducible epoch - const runBuild = () => - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: buildEnv as unknown as NodeJS.ProcessEnv, - stdio: "pipe", - }); - try { - runBuild(); - const firstDigest = digestArtifactTree(r2StagingRoot); - - // The build must actually produce the artifacts whose derivation was - // extracted to scripts/lib/ — a broken import would yield empty/missing - // output, which this asserts before the cheaper hash comparison. - for (const relativePath of [ - "endpoints.json", - "rpc-endpoints.json", - "economics.json", - "endpoint-pools.json", - "endpoint-incidents.json", - ]) { - const artifact = readArtifact(relativePath); - assert.ok( - artifact && typeof artifact === "object", - `${relativePath} should build to a non-empty object`, - ); - } - - runBuild(); - const secondDigest = digestArtifactTree(r2StagingRoot); - - assert.equal( - secondDigest, - firstDigest, - "two consecutive builds must emit a byte-identical R2 staging tree", - ); - } finally { - runBuild(); - restoreSupportArtifacts(supportArtifacts); - } -}, 30_000); - -test("artifact build preserves committed schema index without R2 schema details", () => { - const schemaIndexPath = artifactFilePath("schemas/index.json"); - const originalSchemaIndex = readFileSync(schemaIndexPath, "utf8"); - const originalSchemaIndexJson = JSON.parse(originalSchemaIndex); - const supportArtifacts = snapshotSupportArtifacts(); - const backupDir = mkdtempSync(`${tmpdir()}/metagraphed-schema-r2-`); - const stagingBackup = `${backupDir}/metagraph-r2`; - const hadStagingRoot = existsSync(r2StagingRoot); - if (hadStagingRoot) { - cpSync(r2StagingRoot, stagingBackup, { recursive: true }); - } - - assert.equal(originalSchemaIndexJson.source, "openapi-snapshot"); - assert.equal(originalSchemaIndexJson.schemas.length > 0, true); - - try { - rmSync(r2StagingRoot, { recursive: true, force: true }); - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - - const rebuiltSchemaIndex = readFileSync(schemaIndexPath, "utf8"); - assert.deepEqual(JSON.parse(rebuiltSchemaIndex), originalSchemaIndexJson); - } finally { - writeFileSync(schemaIndexPath, originalSchemaIndex); - rmSync(r2StagingRoot, { recursive: true, force: true }); - if (hadStagingRoot) { - cpSync(stagingBackup, r2StagingRoot, { recursive: true }); - } - restoreSupportArtifacts(supportArtifacts); - rmSync(backupDir, { recursive: true, force: true }); - } -}, 30_000); - -test("artifact build accepts an OpenAPI-vendor JSON content-type for a captured schema entry", () => { - const schemaIndexPath = artifactFilePath("schemas/index.json"); - const originalSchemaIndex = readFileSync(schemaIndexPath, "utf8"); - const supportArtifacts = snapshotSupportArtifacts(); - const schemaIndex = JSON.parse(originalSchemaIndex); - const indexTarget = schemaIndex.schemas?.find( - (schema: Row) => schema.status === "captured", - ); - assert(indexTarget, "expected a captured schema index entry to retype"); - - // A real subnet (SN-71 Leadpoet) serves its OpenAPI document as - // application/vnd.oai.openapi+json rather than plain application/json -- a - // spec-valid, OAI-registered media type. schemaIndexEntryMatchesSurface used - // to require an exact "application/json" match, so this one entry failed the - // reconciler's forgery/staleness guard and wholesale-discarded the entire - // committed index down to an empty placeholder (metagraphed#6411). - indexTarget.content_type = "application/vnd.oai.openapi+json; charset=utf-8"; - - try { - writeFileSync(schemaIndexPath, `${JSON.stringify(schemaIndex, null, 2)}\n`); - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - - const rebuiltSchemaIndex = JSON.parse( - readFileSync(schemaIndexPath, "utf8"), - ); - assert.equal(rebuiltSchemaIndex.source, "openapi-snapshot"); - const rebuiltTarget = rebuiltSchemaIndex.schemas.find( - (schema: Row) => schema.surface_id === indexTarget.surface_id, - ); - assert.equal(rebuiltTarget?.content_type, indexTarget.content_type); - assert.equal(rebuiltTarget?.hash, indexTarget.hash); - } finally { - writeFileSync(schemaIndexPath, originalSchemaIndex); - execFileSync(process.execPath, ["scripts/build-artifacts.ts"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - restoreSupportArtifacts(supportArtifacts); - } -}, 30_000); - test("committed R2 manifest does not use fallback history keys", () => { // Read the git-committed manifest, not the working-tree copy: the Validate // test/checks jobs run `npm run build` before the suite, which regenerates @@ -2531,7 +2171,28 @@ test("R2-only generated artifacts stay out of the public git tree", () => { } }); +// The R2 upload tests below consume the sandbox's STAGING manifest +// (dist/metagraph-r2/metagraph/r2-manifest.json). They must regenerate it +// rather than assume it: the llms.txt test earlier in this file runs the real +// build, which rm's + repopulates the staging dir WITHOUT a manifest (the +// manifest is r2-manifest.ts's product, not the build's). Before the +// build-running tests were split into their own files, their cleanup happened +// to recreate it -- an in-file ordering dependency this helper replaces with +// self-sufficiency. Memoized: one regeneration covers every consumer. +let stagingManifestFresh = false; +function ensureStagingManifest() { + if (stagingManifestFresh) return; + execFileSync(process.execPath, ["scripts/r2-manifest.ts", "--write"], { + cwd: sandbox.scriptCwd, + encoding: "utf8", + env: sandbox.env, + stdio: "pipe", + }); + stagingManifestFresh = true; +} + test("R2 history upload deduplicates content-addressed objects that already exist", async () => { + ensureStagingManifest(); const execFileAsync = promisify(execFile); const manifestPath = path.join(r2StagingRoot, "r2-manifest.json"); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); @@ -2648,6 +2309,7 @@ test("R2 history upload deduplicates content-addressed objects that already exis }, 30_000); test("limited R2 upload dry run skips control manifests", () => { + ensureStagingManifest(); const output = execFileSync( process.execPath, ["scripts/r2-upload.ts", "--dry-run"], @@ -2753,30 +2415,6 @@ function latestArtifactDate(relativePath: string) { .at(-1); } -function snapshotSupportArtifacts(): Map { - return new Map( - SUPPORT_ARTIFACT_PATHS.map((filePath): [string, string] => [ - filePath, - readFileSync(filePath, "utf8"), - ]), - ); -} - -function restoreSupportArtifacts(snapshot: Map) { - for (const [filePath, content] of snapshot) { - writeFileSync(filePath, content); - } - execFileSync(process.execPath, ["scripts/r2-manifest.ts", "--write"], { - cwd: sandbox.scriptCwd, - encoding: "utf8", - env: sandbox.env, - stdio: "pipe", - }); - for (const [filePath, content] of snapshot) { - writeFileSync(filePath, content); - } -} - test("#745 social accounts stay display-only and never feed completeness", () => { const index = JSON.parse( readFileSync(artifactFilePath("subnets.json"), "utf8"), diff --git a/tests/helpers/artifact-build-harness.ts b/tests/helpers/artifact-build-harness.ts new file mode 100644 index 0000000000..6dc29ace35 --- /dev/null +++ b/tests/helpers/artifact-build-harness.ts @@ -0,0 +1,201 @@ +// Shared setup for the test files that run the REAL scripts/build-artifacts.ts. +// +// Extracted from tests/artifacts.test.ts when the build-running tests were +// split across several files (#8937 follow-up). Those tests dominate CI: six of +// them ran a full build, 7.2s to 24.7s each, 99.4s of the file's 118.8s -- and +// that file was 118.8s of a 144.3s pass, so it alone set the floor for the whole +// shared-registry run. vitest parallelizes across FILES and never within one, +// and test.concurrent would not have helped either: these builds go through +// execFileSync, which blocks the worker's event loop, so same-file concurrency +// still serializes them. Sibling files land on separate workers, which is the +// only arrangement that actually overlaps them. +// +// Each file needs its own sandbox -- a sandbox is per-file state, and two builds +// writing one tree is the race #8937 removed -- so this is a factory rather than +// a module-level singleton. +// +// The extra clone per file is ~0.6s against a ~20s build -- the trade is +// overwhelmingly worth it. + +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import { afterAll, beforeAll } from "vitest"; + +import { + artifactDirectoryPath as realArtifactDirectoryPath, + artifactFilePath as realArtifactFilePath, + repoRoot as realRepoRoot, +} from "../../scripts/lib.ts"; +import { createRepoSandbox } from "./repo-sandbox.ts"; + +export interface ArtifactBuildHarness { + /** The sandbox root — every expected path is built from this. */ + root: string; + /** Re-roots an absolute real-repo path into the sandbox. */ + toSandbox: (real: string) => string; + /** lib.ts's artifactFilePath, answered against the sandbox. */ + artifactFilePath: ( + relativePath: string, + options?: { allowPublicFallback?: boolean }, + ) => string; + /** lib.ts's artifactDirectoryPath, answered against the sandbox. */ + artifactDirectoryPath: (relativePath: string) => string; + publicMetagraphRoot: string; + r2StagingRoot: string; + /** The sandbox's served public/ tree. */ + publicTree: string; + /** cwd for spawning the real scripts — the REAL repo (only data is redirected). */ + scriptCwd: string; + /** Env carrying METAGRAPH_REPO_ROOT, for child processes. */ + env: NodeJS.ProcessEnv; + /** Runs a real script against the sandbox's data via METAGRAPH_REPO_ROOT. */ + runNode: (script: string) => void; + /** Parsed JSON of a sandbox artifact. */ + readArtifact: (relativePath: string) => Record; + /** Every file under `dir`, recursively. */ + walkFiles: (dir: string) => string[]; + /** + * Snapshot the committed support artifacts a forged rebuild would overwrite, + * so the build's own output can be rolled back afterwards. + */ + snapshotSupportArtifacts: () => Map; + restoreSupportArtifacts: (snapshot: Map) => void; +} + +function walkFilesRecursive(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.push(...walkFilesRecursive(full)); + } else if (entry.isFile()) { + out.push(full); + } + } + return out; +} + +/** + * Builds a per-FILE sandbox plus the path helpers and script runner the + * build-running tests need, and registers the hooks that keep the sandbox's + * public/ tree byte-stable across the file and delete the sandbox afterwards. + * + * Call once at module scope. `label` only names the temp directory. + * + * The path helpers re-root lib.ts's real answers rather than reimplementing + * them: artifactFilePath's public-vs-R2 tier routing is real logic, and the + * sandbox is a byte copy taken moments earlier, so lib's existsSync-based + * routing resolves identically in both trees. + */ +export function createArtifactBuildHarness( + label: string, +): ArtifactBuildHarness { + const sandbox = createRepoSandbox(label); + const toSandbox = (real: string) => + path.join(sandbox.root, path.relative(realRepoRoot, real)); + const publicTree = path.join(sandbox.root, "public"); + + // Snapshot/restore the served public/ tree so the build-running tests leave it + // exactly as they found it — build-artifacts.ts regenerates from current + // source, which drifts from the committed seed, so restoring exact bytes keeps + // repeated runs within a file idempotent. + const snapshotPublicTree = (): Map => { + if (!existsSync(publicTree)) return new Map(); + return new Map( + walkFilesRecursive(publicTree).map((file: string): [string, Buffer] => [ + file, + readFileSync(file), + ]), + ); + }; + const restorePublicTree = (snapshot: Map) => { + if (existsSync(publicTree)) { + for (const file of walkFilesRecursive(publicTree)) { + if (!snapshot.has(file)) rmSync(file); + } + } + for (const [file, bytes] of snapshot) { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, bytes); + } + }; + + let publicTreeSnapshot: Map; + beforeAll(() => { + publicTreeSnapshot = snapshotPublicTree(); + }); + afterAll(() => { + restorePublicTree(publicTreeSnapshot); + sandbox.cleanup(); + }); + + const artifactFilePath = ( + relativePath: string, + options?: { allowPublicFallback?: boolean }, + ) => toSandbox(realArtifactFilePath(relativePath, options)); + + // Sandbox-rooted, NOT repo-relative. A bare relative path resolves against + // process.cwd() — the real repo — so this would truncate and rewrite the + // committed public/metagraph/r2-manifest.json while the rest of the suite + // reads it, non-atomically (#8937 follow-up). + const supportArtifactPaths = [ + path.join(sandbox.root, "public/metagraph/r2-manifest.json"), + ]; + + return { + root: sandbox.root, + toSandbox, + artifactFilePath, + artifactDirectoryPath: (relativePath) => + toSandbox(realArtifactDirectoryPath(relativePath)), + publicMetagraphRoot: sandbox.publicMetagraphRoot, + r2StagingRoot: sandbox.r2StagingRoot, + publicTree, + scriptCwd: sandbox.scriptCwd, + env: sandbox.env, + readArtifact: (relativePath: string) => + JSON.parse(readFileSync(artifactFilePath(relativePath), "utf8")), + walkFiles: walkFilesRecursive, + snapshotSupportArtifacts: () => + new Map( + supportArtifactPaths.map((filePath): [string, string] => [ + filePath, + readFileSync(filePath, "utf8"), + ]), + ), + restoreSupportArtifacts: (snapshot: Map) => { + for (const [filePath, content] of snapshot) { + writeFileSync(filePath, content); + } + execFileSync(process.execPath, ["scripts/r2-manifest.ts", "--write"], { + cwd: sandbox.scriptCwd, + encoding: "utf8", + env: sandbox.env, + stdio: "pipe", + }); + for (const [filePath, content] of snapshot) { + writeFileSync(filePath, content); + } + }, + runNode: (script: string) => { + execFileSync(process.execPath, [script], { + cwd: sandbox.scriptCwd, + encoding: "utf8", + stdio: "pipe", + // The committed artifacts are an inert cold-start seed (ADR 0006) that + // drifts from live source between publishes. This suite validates + // structure; committed-vs-fresh freshness parity is gated in CI + // (post-build) instead. + env: { ...sandbox.env, METAGRAPH_ALLOW_SEED_DRIFT: "1" }, + }); + }, + }; +}