diff --git a/docs/UAT.md b/docs/UAT.md index 05fc46c..cdebd01 100644 --- a/docs/UAT.md +++ b/docs/UAT.md @@ -163,7 +163,14 @@ Test at least one valid result from each available class: - [`provider-matrix/signed-redirect`] Provider-proxied/signed URL with bounded redirects. - [`provider-matrix/subtitle-text`] Text subtitle provider. - [`provider-matrix/subtitle-ass`] ASS/SSA subtitle provider. -- [`provider-matrix/subtitle-vobsub`] VobSub archive source when available. +- [`provider-matrix/subtitle-vobsub`] Use the deterministic Jumpgate VobSub fixture on + both devices. Capture the exact visible bitmap text `JUMPGATE VOBSUB 1`, `2`, and `3` + while the media progress rail shows the respective 2-5, 7-10, and 12-15 second + windows. Before recording each capture, inspect it and explicitly attest that the exact + cue/time rail are visible and that no notification, account label, pairing code, private + URL, QR code, or unrelated screen content is present. The six captures across both devices + must be distinct. Evidence PNGs are canonical RGB-only files without ancillary/private chunks + or alpha channels. A delivered IDX/SUB pair without visible rendering is a failure. For each source, record whether canonical identity was claimed. Transport success and identity success are separate results. @@ -253,7 +260,9 @@ Use Trakt's account activity and sanitized device logs to verify: - [`subtitles/text-ass-fidelity`] Text and ASS/SSA content retains expected text, timing, encoding, and styling. - [`subtitles/vobsub-atomic-pair`] VobSub publishes a complete matching IDX/SUB pair before - Kodi injection. + Kodi injection. The finalized report must hash-bind the three PNG render captures; + they are committed beside the report as `-vobsub-cue-1.png` through + `-vobsub-cue-3.png`. - [`subtitles/picker-controls`] Subtitle picker, enable/disable, language selection, and delay controls work. - [`subtitles/replacement-generation`] Replacing a subtitle removes the previous generation diff --git a/release/evidence/README.md b/release/evidence/README.md index 9d4d593..12cccf9 100644 --- a/release/evidence/README.md +++ b/release/evidence/README.md @@ -26,11 +26,27 @@ npm run uat:evidence -- init --device-class tv --manufacturer Google \ npm run uat:evidence -- record --workbook .uat/tv.json \ --case lifecycle/start-first-frames \ --observation "First rendered frames appeared and playback remained responsive." +npm run uat:evidence -- record-vobsub --workbook .uat/tv.json \ + --cue 1 --capture release/evidence/tv-vobsub-cue-1.png \ + --visual-review cue-and-time-rail-confirmed \ + --privacy-review sanitized-for-publication npm run uat:evidence -- status --workbook .uat/tv.json npm run uat:evidence -- finalize --workbook .uat/tv.json \ --output release/evidence/tv.json ``` +Each device report requires three PNG captures named +`-vobsub-cue-1.png` through `-vobsub-cue-3.png` beside the finalized +JSON report. Capture each exact cue while the fixture's visible progress rail lies in +its locked media-time window. Inspect each capture before providing both required review +attestations: the exact cue and time rail must be visible, and the image must be safe to +publish. The recorder rejects duplicate captures, malformed or undecodable PNGs, every ancillary +or private chunk, alpha channels, non-canonical input paths, and captures without both +attestations. All six phone/TV captures must have distinct hashes. Release validation +downloads the exact paths from the report's immutable commit and repeats structure and +hash verification. Do not include notifications, account labels, pairing codes, private +URLs, QR codes, or other unrelated screen content. + Commit the finalized phone and TV reports first. In a second commit, build `physical-uat.json` using immutable blob URLs at that first commit SHA: @@ -43,8 +59,9 @@ npm run uat:evidence -- index \ --output release/evidence/physical-uat.json ``` -`evidenceUrl` must be an immutable public blob URL in a Jumpgate repository at a full -commit SHA. The validator downloads that blob and verifies `evidenceSha256`; issue pages, +`evidenceUrl` must be the canonical `release/evidence/phone.json` or +`release/evidence/tv.json` immutable blob URL in the Jumpgate repository at a full commit +SHA. The validator downloads that blob and verifies `evidenceSha256`; issue pages, branch URLs, expiring Action artifacts, query strings, and fragments are rejected. Serial numbers, account names, private URLs, tokens, logs, pairing data, and QR images are not allowed in either the index or the evidence blob. diff --git a/release/png-evidence.mjs b/release/png-evidence.mjs new file mode 100644 index 0000000..d003876 --- /dev/null +++ b/release/png-evidence.mjs @@ -0,0 +1,97 @@ +import { inflateSync } from "node:zlib"; + +const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +const ALLOWED_CHUNKS = new Set(["IHDR", "IDAT", "IEND"]); +const CRC_TABLE = Array.from({ length: 256 }, (_, value) => { + let crc = value; + for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + return crc >>> 0; +}); + +function fail(message) { + throw new Error(message); +} + +function crc32(bytes) { + let crc = 0xffffffff; + for (const byte of bytes) crc = CRC_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); + return (crc ^ 0xffffffff) >>> 0; +} + +export function validatePublicEvidencePng(bytes, maximumBytes = 20 * 1024 * 1024) { + if (!Buffer.isBuffer(bytes) || bytes.length < 57 || bytes.length > maximumBytes) { + fail("VobSub cue capture must be a bounded complete PNG file"); + } + if (!bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)) { + fail("VobSub cue capture must be a PNG file"); + } + + let offset = PNG_SIGNATURE.length; + let width = 0; + let height = 0; + let channels = 0; + let sawIhdr = false; + let sawIdat = false; + let sawIend = false; + const compressed = []; + + while (offset < bytes.length) { + if (offset + 12 > bytes.length) fail("VobSub cue capture has a truncated PNG chunk"); + const length = bytes.readUInt32BE(offset); + const end = offset + 12 + length; + if (end > bytes.length) fail("VobSub cue capture has a truncated PNG chunk"); + const typeBytes = bytes.subarray(offset + 4, offset + 8); + const type = typeBytes.toString("ascii"); + if (!/^[A-Za-z]{4}$/.test(type)) fail("VobSub cue capture has an invalid PNG chunk type"); + const data = bytes.subarray(offset + 8, offset + 8 + length); + const expectedCrc = bytes.readUInt32BE(offset + 8 + length); + if (crc32(Buffer.concat([typeBytes, data])) !== expectedCrc) { + fail(`VobSub cue capture has an invalid ${type} checksum`); + } + if (!ALLOWED_CHUNKS.has(type)) fail(`VobSub cue capture must not contain ancillary or private PNG chunk ${type}`); + + if (!sawIhdr) { + if (type !== "IHDR" || length !== 13) fail("VobSub cue capture must begin with IHDR"); + width = data.readUInt32BE(0); + height = data.readUInt32BE(4); + const bitDepth = data[8]; + const colorType = data[9]; + if (width < 320 || width > 7680 || height < 240 || height > 4320) { + fail("VobSub cue capture dimensions are outside the public evidence bounds"); + } + if (bitDepth !== 8 || colorType !== 2 || data[10] !== 0 || data[11] !== 0 || data[12] !== 0) { + fail("VobSub cue capture must be a non-interlaced 8-bit RGB PNG"); + } + channels = 3; + sawIhdr = true; + } else if (type === "IHDR") { + fail("VobSub cue capture must contain exactly one IHDR chunk"); + } else if (type === "IDAT") { + if (sawIend) fail("VobSub cue capture contains image data after IEND"); + sawIdat = true; + compressed.push(data); + } else if (type === "IEND") { + if (length !== 0 || !sawIdat || sawIend) fail("VobSub cue capture has an invalid IEND chunk"); + sawIend = true; + offset = end; + if (offset !== bytes.length) fail("VobSub cue capture has trailing bytes after IEND"); + break; + } + offset = end; + } + + if (!sawIhdr || !sawIdat || !sawIend) fail("VobSub cue capture is missing required PNG chunks"); + const rowBytes = width * channels; + const expectedLength = height * (rowBytes + 1); + let pixels; + try { + pixels = inflateSync(Buffer.concat(compressed), { maxOutputLength: expectedLength }); + } catch { + fail("VobSub cue capture contains invalid compressed image data"); + } + if (pixels.length !== expectedLength) fail("VobSub cue capture has an invalid decoded image size"); + for (let row = 0; row < height; row += 1) { + if (pixels[row * (rowBytes + 1)] > 4) fail("VobSub cue capture has an invalid PNG row filter"); + } + return { width, height }; +} diff --git a/release/tests/png-fixture.mjs b/release/tests/png-fixture.mjs new file mode 100644 index 0000000..c643105 --- /dev/null +++ b/release/tests/png-fixture.mjs @@ -0,0 +1,45 @@ +import { deflateSync } from "node:zlib"; + +const SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +const CRC_TABLE = Array.from({ length: 256 }, (_, value) => { + let crc = value; + for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + return crc >>> 0; +}); + +function crc32(bytes) { + let crc = 0xffffffff; + for (const byte of bytes) crc = CRC_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); + return (crc ^ 0xffffffff) >>> 0; +} + +function chunk(type, data = Buffer.alloc(0)) { + const typeBytes = Buffer.from(type, "ascii"); + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(Buffer.concat([typeBytes, data]))); + return Buffer.concat([length, typeBytes, data, crc]); +} + +export function evidencePng(marker = 1, extraChunks = []) { + const width = 320; + const height = 240; + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); + ihdr.writeUInt32BE(height, 4); + ihdr.set([8, 2, 0, 0, 0], 8); + const rows = Buffer.alloc(height * (1 + width * 3)); + for (let row = 0; row < height; row += 1) { + const offset = row * (1 + width * 3); + rows[offset] = 0; + rows.fill((marker + row) & 0xff, offset + 1, offset + 1 + width * 3); + } + return Buffer.concat([ + SIGNATURE, + chunk("IHDR", ihdr), + ...extraChunks.map(({ type, data }) => chunk(type, data)), + chunk("IDAT", deflateSync(rows)), + chunk("IEND"), + ]); +} diff --git a/release/tests/uat-evidence.test.mjs b/release/tests/uat-evidence.test.mjs index 8498386..1ef00e0 100644 --- a/release/tests/uat-evidence.test.mjs +++ b/release/tests/uat-evidence.test.mjs @@ -8,6 +8,7 @@ import { createWorkbook, finalizeWorkbook, recordPass, + recordVobSubCue, validateWorkbook, } from "../uat-evidence.mjs"; import { @@ -15,6 +16,7 @@ import { requiredUatCasesForDevice, validateEvidence, } from "../validate-release.mjs"; +import { evidencePng } from "./png-fixture.mjs"; const candidate = JSON.parse(readFileSync(new URL("../candidate.json", import.meta.url), "utf8")); const now = new Date("2026-08-13T12:00:00Z"); @@ -31,6 +33,13 @@ function input(deviceClass = "tv") { function completedReport(deviceClass) { let workbook = createWorkbook(candidate, input(deviceClass), now); + for (const cue of [1, 2, 3]) workbook = recordVobSubCue( + workbook, + candidate, + cue, + evidencePng(cue + (deviceClass === "tv" ? 10 : 0)), + { capturePath: `release/evidence/${deviceClass}-vobsub-cue-${cue}.png`, visualReview: "cue-and-time-rail-confirmed", privacyReview: "sanitized-for-publication" }, + ); for (const [index, id] of requiredUatCasesForDevice(deviceClass).entries()) { workbook = recordPass(workbook, candidate, id, `Observed expected behavior for policy case ${index + 1}.`); } @@ -46,6 +55,39 @@ test("workbooks bind public candidate artifacts and exact device-scoped policy c assert.throws(() => finalizeWorkbook(workbook, candidate, now), /remain pending/); }); +test("VobSub render evidence requires three bounded PNG captures", () => { + let workbook = createWorkbook(candidate, input(), now); + assert.throws( + () => recordVobSubCue(workbook, candidate, 1, Buffer.from("not a PNG")), + /PNG file/, + ); + assert.throws( + () => recordVobSubCue(workbook, candidate, 1, evidencePng(1)), + /visual-review/, + ); + assert.throws( + () => recordVobSubCue(workbook, candidate, 1, evidencePng(1), { capturePath: "elsewhere.png", visualReview: "cue-and-time-rail-confirmed", privacyReview: "sanitized-for-publication" }), + /must be read from release\/evidence/, + ); + const review = (cue) => ({ capturePath: `release/evidence/tv-vobsub-cue-${cue}.png`, visualReview: "cue-and-time-rail-confirmed", privacyReview: "sanitized-for-publication" }); + workbook = recordVobSubCue(workbook, candidate, 1, evidencePng(1), review(1)); + assert.throws( + () => recordVobSubCue(workbook, candidate, 2, evidencePng(1), review(2)), + /distinct images/, + ); + assert.throws( + () => recordVobSubCue(workbook, candidate, 2, evidencePng(2, [{ type: "tEXt", data: Buffer.from("Account=test") }]), review(2)), + /ancillary or private/, + ); + for (const cue of [2, 3]) workbook = recordVobSubCue(workbook, candidate, cue, evidencePng(cue), review(cue)); + workbook = recordVobSubCue(workbook, candidate, 3, evidencePng(3), review(3)); + assert.deepEqual(workbook.vobsubRenderEvidence.map(({ cue, status }) => ({ cue, status })), [ + { cue: 1, status: "pass" }, + { cue: 2, status: "pass" }, + { cue: 3, status: "pass" }, + ]); +}); + test("recording rejects unknown cases and secret-shaped observations", () => { const workbook = createWorkbook(candidate, input(), now); assert.throws( diff --git a/release/tests/validate-release.test.mjs b/release/tests/validate-release.test.mjs index c4141ff..fe6a9cd 100644 --- a/release/tests/validate-release.test.mjs +++ b/release/tests/validate-release.test.mjs @@ -41,7 +41,9 @@ import { verifyGithubDeploymentAttestation, verifyCurrentPublicRefManifest, verifyLiveBridgeState, + verifyEvidenceArtifacts, } from "../validate-release.mjs"; +import { evidencePng } from "./png-fixture.mjs"; const CANDIDATE_TEMPLATE = JSON.parse( readFileSync(new URL("../candidate.json", import.meta.url), "utf8"), @@ -86,6 +88,7 @@ function physicalEvidence() { evidenceUrl: `https://github.com/ruizkinio/Jumpgate/blob/${marker.repeat(40)}/release/evidence/${deviceClass}.json`, caseCount: requiredUatCasesForDevice(deviceClass).length, + vobsubCaptureSha256: [1, 2, 3].map((cue) => `${deviceClass === "phone" ? cue : cue + 3}`.repeat(64)), }; }; return { @@ -105,8 +108,9 @@ function physicalEvidence() { } function uatReport(run, locked = candidate()) { + const captureHashes = run.vobsubCaptureSha256; return { - schemaVersion: 3, + schemaVersion: 4, candidate: { coordinatedVersion: locked.coordinatedVersion, bridgeCommit: locked.components.bridge.commit, @@ -134,6 +138,11 @@ function uatReport(run, locked = candidate()) { buildSha: locked.components.bridge.commit, imageDigest: locked.components.bridge.imageDigest, }, + vobsubRenderEvidence: [ + { cue: 1, text: "JUMPGATE VOBSUB 1", windowStartMs: 2_000, windowEndMs: 5_000, status: "pass", sha256: captureHashes[0], capturePath: `release/evidence/${run.deviceClass}-vobsub-cue-1.png`, visualReview: "cue-and-time-rail-confirmed", privacyReview: "sanitized-for-publication" }, + { cue: 2, text: "JUMPGATE VOBSUB 2", windowStartMs: 7_000, windowEndMs: 10_000, status: "pass", sha256: captureHashes[1], capturePath: `release/evidence/${run.deviceClass}-vobsub-cue-2.png`, visualReview: "cue-and-time-rail-confirmed", privacyReview: "sanitized-for-publication" }, + { cue: 3, text: "JUMPGATE VOBSUB 3", windowStartMs: 12_000, windowEndMs: 15_000, status: "pass", sha256: captureHashes[2], capturePath: `release/evidence/${run.deviceClass}-vobsub-cue-3.png`, visualReview: "cue-and-time-rail-confirmed", privacyReview: "sanitized-for-publication" }, + ], cases: requiredUatCasesForDevice(run.deviceClass).map((id) => ({ id, status: "pass", @@ -691,11 +700,12 @@ test("physical evidence requires fresh distinct devices and locked ABI artifacts ); const shared = physicalEvidence(); - shared.runs[1].evidenceUrl = shared.runs[0].evidenceUrl; + shared.runs[1].manufacturer = shared.runs[0].manufacturer; + shared.runs[1].model = shared.runs[0].model; shared.runs[1].evidenceSha256 = shared.runs[0].evidenceSha256; assert.throws( () => validateEvidence(shared, candidate(), TEST_NOW, TEST_RELEASE_SIGNER_POLICY), - /distinct immutable/, + /different physical devices/, ); const wrongSigner = physicalEvidence(); @@ -777,10 +787,31 @@ test("UAT reports require an observed pass for every device-scoped protocol case assert.throws(() => validateUatReport(tvOnly, evidence.runs[0], candidate()), /required for phone/); const oldSchema = structuredClone(report); - oldSchema.schemaVersion = 2; + oldSchema.schemaVersion = 3; assert.throws( () => validateUatReport(oldSchema, evidence.runs[0], candidate()), - /schemaVersion must be 3/, + /schemaVersion must be 4/, + ); + + const reusedCapture = physicalEvidence(); + reusedCapture.runs[1].vobsubCaptureSha256[0] = reusedCapture.runs[0].vobsubCaptureSha256[0]; + assert.throws( + () => validateEvidence(reusedCapture, candidate(), TEST_NOW, TEST_RELEASE_SIGNER_POLICY), + /six distinct VobSub/, + ); + + const missingCapture = structuredClone(report); + missingCapture.vobsubRenderEvidence.pop(); + assert.throws( + () => validateUatReport(missingCapture, evidence.runs[0], candidate()), + /exactly three VobSub render captures/, + ); + + const wrongCue = structuredClone(report); + wrongCue.vobsubRenderEvidence[1].text = "Wrong cue"; + assert.throws( + () => validateUatReport(wrongCue, evidence.runs[0], candidate()), + /must be "JUMPGATE VOBSUB 2"/, ); const control = structuredClone(report); @@ -806,6 +837,39 @@ test("evidence URLs must be immutable public blobs without ambient data", () => ); }); +test("immutable VobSub capture verification enforces exact paths, bytes, and PNG structure", async () => { + const locked = candidate(); + const evidence = physicalEvidence(); + const reports = new Map(); + for (const run of evidence.runs) { + const report = uatReport(run, locked); + for (const capture of report.vobsubRenderEvidence) { + const bytes = evidencePng(capture.cue + (run.deviceClass === "tv" ? 10 : 0)); + capture.sha256 = createHash("sha256").update(bytes).digest("hex"); + reports.set(capture.capturePath, bytes); + } + run.vobsubCaptureSha256 = report.vobsubRenderEvidence.map((capture) => capture.sha256); + const reportBytes = Buffer.from(`${JSON.stringify(report)}\n`); + run.evidenceSha256 = createHash("sha256").update(reportBytes).digest("hex"); + reports.set(`release/evidence/${run.deviceClass}.json`, reportBytes); + } + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const path = new URL(url).pathname.split("/").slice(4).join("/"); + const bytes = reports.get(path); + return bytes + ? new Response(bytes, { status: 200, headers: { "content-length": String(bytes.length) } }) + : new Response("missing", { status: 404 }); + }; + try { + await verifyEvidenceArtifacts(evidence, locked); + reports.set("release/evidence/phone-vobsub-cue-2.png", Buffer.from("not png")); + await assert.rejects(() => verifyEvidenceArtifacts(evidence, locked), /digest does not match|complete PNG/); + } finally { + globalThis.fetch = originalFetch; + } +}); + test("security audit evidence is scoped, zero-finding, and reproducible", () => { const emptyAllowlist = securityAllowlist(); const audit = securityAudit(); diff --git a/release/uat-evidence.mjs b/release/uat-evidence.mjs index 5ecb8b7..48ad087 100644 --- a/release/uat-evidence.mjs +++ b/release/uat-evidence.mjs @@ -13,8 +13,15 @@ import { validateEvidence, validateUatReport, } from "./validate-release.mjs"; +import { validatePublicEvidencePng } from "./png-evidence.mjs"; -const WORKBOOK_SCHEMA_VERSION = 2; +const WORKBOOK_SCHEMA_VERSION = 3; +const UAT_REPORT_SCHEMA_VERSION = 4; +export const VOBSUB_RENDER_CUES = Object.freeze([ + Object.freeze({ cue: 1, text: "JUMPGATE VOBSUB 1", windowStartMs: 2_000, windowEndMs: 5_000 }), + Object.freeze({ cue: 2, text: "JUMPGATE VOBSUB 2", windowStartMs: 7_000, windowEndMs: 10_000 }), + Object.freeze({ cue: 3, text: "JUMPGATE VOBSUB 3", windowStartMs: 12_000, windowEndMs: 15_000 }), +]); const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true }); const OBSERVATION_REJECTIONS = [ [/(?:https?|stremio):\/\//i, "URLs"], @@ -106,6 +113,52 @@ export function assertSanitizedObservation(observation) { return observation; } +function emptyVobSubRenderEvidence() { + return VOBSUB_RENDER_CUES.map((cue) => ({ + ...cue, + status: "pending", + sha256: "", + capturePath: "", + visualReview: "pending", + privacyReview: "pending", + })); +} + +function validateVobSubRenderEvidence(entries, deviceClass, allowPending) { + if (!Array.isArray(entries) || entries.length !== VOBSUB_RENDER_CUES.length) { + fail("VobSub render evidence must contain exactly three cue captures"); + } + for (const [index, entry] of entries.entries()) { + assertExactKeys( + entry, + ["cue", "text", "windowStartMs", "windowEndMs", "status", "sha256", "capturePath", "visualReview", "privacyReview"], + `VobSub cue ${index + 1}`, + ); + const expected = VOBSUB_RENDER_CUES[index]; + assertSame( + { cue: entry.cue, text: entry.text, windowStartMs: entry.windowStartMs, windowEndMs: entry.windowEndMs }, + expected, + `VobSub cue ${index + 1}`, + ); + if ( + entry.status === "pending" && allowPending && entry.sha256 === "" && + entry.capturePath === "" && entry.visualReview === "pending" && entry.privacyReview === "pending" + ) continue; + if (entry.status !== "pass" || !/^[a-f0-9]{64}$/.test(entry.sha256)) { + fail(`VobSub cue ${index + 1} requires a passed PNG capture SHA-256`); + } + if (entry.capturePath !== `release/evidence/${deviceClass}-vobsub-cue-${index + 1}.png`) { + fail(`VobSub cue ${index + 1} capture path is not canonical`); + } + if (entry.visualReview !== "cue-and-time-rail-confirmed" || entry.privacyReview !== "sanitized-for-publication") { + fail(`VobSub cue ${index + 1} requires visual and privacy review attestations`); + } + } + const hashes = entries.filter((entry) => entry.status === "pass").map((entry) => entry.sha256); + if (new Set(hashes).size !== hashes.length) fail("VobSub cue captures must be distinct images"); + return entries; +} + export function createWorkbook(candidate, input, now = new Date()) { validateCandidate(candidate); const device = validateDeviceInput(input, candidate); @@ -115,6 +168,7 @@ export function createWorkbook(candidate, input, now = new Date()) { device, bridge: bridgeRecord(candidate), createdAt: now.toISOString(), + vobsubRenderEvidence: emptyVobSubRenderEvidence(), cases: requiredUatCasesForDevice(device.deviceClass).map((id) => ({ id, status: "pending", @@ -125,7 +179,7 @@ export function createWorkbook(candidate, input, now = new Date()) { export function validateWorkbook(workbook, candidate) { validateCandidate(candidate); - assertExactKeys(workbook, ["schemaVersion", "candidate", "device", "bridge", "createdAt", "cases"], "workbook"); + assertExactKeys(workbook, ["schemaVersion", "candidate", "device", "bridge", "createdAt", "vobsubRenderEvidence", "cases"], "workbook"); if (workbook.schemaVersion !== WORKBOOK_SCHEMA_VERSION) fail("unsupported workbook schemaVersion"); assertSame(workbook.candidate, candidateRecord(candidate), "workbook candidate"); assertSame(workbook.bridge, bridgeRecord(candidate), "workbook Bridge"); @@ -133,6 +187,7 @@ export function validateWorkbook(workbook, candidate) { assertExactKeys(workbook.device, Object.keys(expectedDevice), "workbook device"); assertSame(workbook.device, expectedDevice, "workbook device"); if (Number.isNaN(new Date(workbook.createdAt).valueOf())) fail("workbook createdAt is invalid"); + validateVobSubRenderEvidence(workbook.vobsubRenderEvidence, workbook.device.deviceClass, true); const requiredCases = requiredUatCasesForDevice(workbook.device.deviceClass); if (!Array.isArray(workbook.cases) || workbook.cases.length !== requiredCases.length) { fail(`workbook must contain every UAT case required for ${workbook.device.deviceClass}`); @@ -147,6 +202,34 @@ export function validateWorkbook(workbook, candidate) { return workbook; } +export function recordVobSubCue(workbook, candidate, cue, captureBytes, { capturePath, visualReview, privacyReview } = {}) { + validateWorkbook(workbook, candidate); + if (!Number.isInteger(cue) || cue < 1 || cue > VOBSUB_RENDER_CUES.length) { + fail("VobSub cue must be 1, 2, or 3"); + } + validatePublicEvidencePng(captureBytes); + if (visualReview !== "cue-and-time-rail-confirmed") { + fail("VobSub cue capture requires --visual-review cue-and-time-rail-confirmed"); + } + if (privacyReview !== "sanitized-for-publication") { + fail("VobSub cue capture requires --privacy-review sanitized-for-publication"); + } + const captureSha256 = sha256(captureBytes); + const canonicalPath = `release/evidence/${workbook.device.deviceClass}-vobsub-cue-${cue}.png`; + if (capturePath !== canonicalPath) fail(`VobSub cue capture must be read from ${canonicalPath}`); + if (updatedVobSubHashes(workbook, cue).has(captureSha256)) fail("VobSub cue captures must be distinct images"); + const updated = structuredClone(workbook); + updated.vobsubRenderEvidence[cue - 1] = { + ...VOBSUB_RENDER_CUES[cue - 1], + status: "pass", + sha256: captureSha256, + capturePath: canonicalPath, + visualReview, + privacyReview, + }; + return validateWorkbook(updated, candidate); +} + export function recordPass(workbook, candidate, caseId, observation) { validateWorkbook(workbook, candidate); assertSanitizedObservation(observation); @@ -162,15 +245,21 @@ export function finalizeWorkbook(workbook, candidate, now = new Date()) { validateWorkbook(workbook, candidate); const pending = workbook.cases.filter((entry) => entry.status !== "pass"); if (pending.length > 0) fail(`cannot finalize: ${pending.length} UAT cases remain pending`); + validateVobSubRenderEvidence(workbook.vobsubRenderEvidence, workbook.device.deviceClass, false); const report = { - schemaVersion: 3, + schemaVersion: UAT_REPORT_SCHEMA_VERSION, candidate: workbook.candidate, device: workbook.device, testedAt: now.toISOString().replace(/\.\d{3}Z$/, "Z"), bridge: workbook.bridge, + vobsubRenderEvidence: workbook.vobsubRenderEvidence, cases: workbook.cases, }; - validateUatReport(report, { ...report.device, testedAt: report.testedAt }, candidate); + validateUatReport(report, { + ...report.device, + testedAt: report.testedAt, + vobsubCaptureSha256: report.vobsubRenderEvidence.map((capture) => capture.sha256), + }, candidate); return report; } @@ -178,12 +267,18 @@ function sha256(bytes) { return createHash("sha256").update(bytes).digest("hex"); } +function updatedVobSubHashes(workbook, replacedCue) { + return new Set(workbook.vobsubRenderEvidence + .filter((entry) => entry.status === "pass" && entry.cue !== replacedCue) + .map((entry) => entry.sha256)); +} + export function createEvidenceIndex(candidate, reports, now = new Date()) { validateCandidate(candidate); if (!Array.isArray(reports) || reports.length !== 2) fail("exactly one phone and one TV report are required"); const runs = reports.map(({ bytes, evidenceUrl }) => { if (!Buffer.isBuffer(bytes)) fail("report bytes must be a Buffer"); - parseEvidenceBlobUrl(evidenceUrl); + const parsed = parseEvidenceBlobUrl(evidenceUrl); let report; try { report = JSON.parse(UTF8_DECODER.decode(bytes)); @@ -196,7 +291,11 @@ export function createEvidenceIndex(candidate, reports, now = new Date()) { evidenceSha256: sha256(bytes), evidenceUrl, caseCount: report.cases?.length, + vobsubCaptureSha256: report.vobsubRenderEvidence?.map((capture) => capture.sha256), }; + if (parsed.repository !== "Jumpgate" || parsed.filePath !== `release/evidence/${run.deviceClass}.json`) { + fail(`evidenceUrl must be the canonical immutable Jumpgate ${run.deviceClass} UAT report URL`); + } validateUatReport(report, run, candidate); return run; }); @@ -231,12 +330,19 @@ function required(values, name) { return value; } +function parseStrictInteger(value, name) { + if (!/^(?:0|[1-9][0-9]*)$/.test(value)) fail(`--${name} must be an integer`); + return Number(value); +} + function cliOptions() { return { candidate: { type: "string", default: resolve(dirname(fileURLToPath(import.meta.url)), "candidate.json") }, output: { type: "string" }, workbook: { type: "string" }, "device-class": { type: "string" }, manufacturer: { type: "string" }, model: { type: "string" }, "android-api": { type: "string" }, abi: { type: "string" }, case: { type: "string" }, observation: { type: "string" }, + cue: { type: "string" }, capture: { type: "string" }, "visual-review": { type: "string" }, + "privacy-review": { type: "string" }, "phone-report": { type: "string" }, "phone-url": { type: "string" }, "tv-report": { type: "string" }, "tv-url": { type: "string" }, }; @@ -263,11 +369,30 @@ export function runCli(args = process.argv.slice(2)) { console.log(`Recorded pass; ${updated.cases.filter((entry) => entry.status === "pending").length} cases remain.`); return; } + if (command === "record-vobsub") { + const path = required(values, "workbook"); + const updated = recordVobSubCue( + readJson(path, "workbook"), + candidate, + parseStrictInteger(required(values, "cue"), "cue"), + readFileSync(required(values, "capture")), + { + capturePath: required(values, "capture").replaceAll("\\", "/"), + visualReview: required(values, "visual-review"), + privacyReview: required(values, "privacy-review"), + }, + ); + writeJsonAtomic(path, updated); + console.log(`Recorded VobSub cue capture ${required(values, "cue")}.`); + return; + } if (command === "status") { const workbook = validateWorkbook(readJson(required(values, "workbook"), "workbook"), candidate); const pending = workbook.cases.filter((entry) => entry.status === "pending"); - console.log(`${workbook.device.deviceClass}: ${workbook.cases.length - pending.length}/${workbook.cases.length} passed.`); + const pendingCaptures = workbook.vobsubRenderEvidence.filter((entry) => entry.status === "pending"); + console.log(`${workbook.device.deviceClass}: ${workbook.cases.length - pending.length}/${workbook.cases.length} cases passed; ${3 - pendingCaptures.length}/3 VobSub captures recorded.`); for (const entry of pending) console.log(entry.id); + for (const entry of pendingCaptures) console.log(`vobsub/cue-${entry.cue}`); return; } if (command === "finalize") { @@ -285,7 +410,7 @@ export function runCli(args = process.argv.slice(2)) { console.log("Created physical UAT evidence index from immutable report bytes."); return; } - fail("command must be init, record, status, finalize, or index"); + fail("command must be init, record, record-vobsub, status, finalize, or index"); } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { diff --git a/release/validate-release.mjs b/release/validate-release.mjs index e4e5932..cd4775b 100644 --- a/release/validate-release.mjs +++ b/release/validate-release.mjs @@ -8,6 +8,8 @@ import { dirname, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { inflateRawSync } from "node:zlib"; +import { validatePublicEvidencePng } from "./png-evidence.mjs"; + export const COMPONENT_POLICIES = Object.freeze({ bridge: Object.freeze({ repository: "https://github.com/ruizkinio/Jumpgate-bridge.git", @@ -409,6 +411,14 @@ export function parseEvidenceBlobUrl(value, path = "evidenceUrl") { return { repository: match[1], commit: match[2], filePath: match[3] }; } +function parseCanonicalUatReportUrl(value, deviceClass, path = "evidenceUrl") { + const blob = parseEvidenceBlobUrl(value, path); + if (blob.repository !== "Jumpgate" || blob.filePath !== `release/evidence/${deviceClass}.json`) { + fail(`${path} must be the canonical immutable Jumpgate ${deviceClass} UAT report URL`); + } + return blob; +} + function validateComponentIdentity(component, policy, path) { if (component.repository !== policy.repository || component.branch !== policy.branch) { fail(`${path} must use ${policy.repository}#${policy.branch}`); @@ -685,6 +695,7 @@ export function validateEvidence( const seenClasses = new Set(); const seenDevices = new Set(); const seenReports = new Set(); + const seenCaptureHashes = new Set(); for (const [index, run] of evidence.runs.entries()) { const path = `evidence.runs[${index}]`; assertExactKeys( @@ -706,6 +717,7 @@ export function validateEvidence( "evidenceSha256", "evidenceUrl", "caseCount", + "vobsubCaptureSha256", ], path, ); @@ -773,8 +785,16 @@ export function validateEvidence( if (run.caseCount !== requiredCases.length) { fail(`${path}.caseCount must cover every UAT case required for ${run.deviceClass}`); } + if (!Array.isArray(run.vobsubCaptureSha256) || run.vobsubCaptureSha256.length !== 3) { + fail(`${path}.vobsubCaptureSha256 must contain exactly three capture hashes`); + } + for (const [captureIndex, hash] of run.vobsubCaptureSha256.entries()) { + assertHash(hash, `${path}.vobsubCaptureSha256[${captureIndex}]`); + if (seenCaptureHashes.has(hash)) fail("phone and TV must use six distinct VobSub render captures"); + seenCaptureHashes.add(hash); + } assertHash(run.evidenceSha256, `${path}.evidenceSha256`); - parseEvidenceBlobUrl(run.evidenceUrl, `${path}.evidenceUrl`); + parseCanonicalUatReportUrl(run.evidenceUrl, run.deviceClass, `${path}.evidenceUrl`); const reportKey = `${run.evidenceUrl}\n${run.evidenceSha256}`; if (seenReports.has(reportKey)) { fail("phone and TV must use distinct immutable evidence reports"); @@ -787,10 +807,10 @@ export function validateEvidence( export function validateUatReport(report, run, candidate) { assertExactKeys( report, - ["schemaVersion", "candidate", "device", "testedAt", "bridge", "cases"], + ["schemaVersion", "candidate", "device", "testedAt", "bridge", "vobsubRenderEvidence", "cases"], "UAT report", ); - if (report.schemaVersion !== 3) fail("UAT report schemaVersion must be 3"); + if (report.schemaVersion !== 4) fail("UAT report schemaVersion must be 4"); assertExactValue(report.candidate, { coordinatedVersion: candidate.coordinatedVersion, bridgeCommit: candidate.components.bridge.commit, @@ -818,6 +838,36 @@ export function validateUatReport(report, run, candidate) { buildSha: candidate.components.bridge.commit, imageDigest: candidate.components.bridge.imageDigest, }, "UAT report.bridge"); + const vobsubCues = [ + { cue: 1, text: "JUMPGATE VOBSUB 1", windowStartMs: 2_000, windowEndMs: 5_000 }, + { cue: 2, text: "JUMPGATE VOBSUB 2", windowStartMs: 7_000, windowEndMs: 10_000 }, + { cue: 3, text: "JUMPGATE VOBSUB 3", windowStartMs: 12_000, windowEndMs: 15_000 }, + ]; + if (!Array.isArray(report.vobsubRenderEvidence) || report.vobsubRenderEvidence.length !== 3) { + fail("UAT report must contain exactly three VobSub render captures"); + } + for (const [index, capture] of report.vobsubRenderEvidence.entries()) { + const path = `UAT report.vobsubRenderEvidence[${index}]`; + assertExactKeys(capture, ["cue", "text", "windowStartMs", "windowEndMs", "status", "sha256", "capturePath", "visualReview", "privacyReview"], path); + assertExactValue(capture, { + ...vobsubCues[index], + status: "pass", + sha256: capture.sha256, + capturePath: `release/evidence/${run.deviceClass}-vobsub-cue-${index + 1}.png`, + visualReview: "cue-and-time-rail-confirmed", + privacyReview: "sanitized-for-publication", + }, path); + assertHash(capture.sha256, `${path}.sha256`); + } + if (new Set(report.vobsubRenderEvidence.map((capture) => capture.sha256)).size !== 3) { + fail("UAT report VobSub render captures must be distinct images"); + } + if ( + !Array.isArray(run.vobsubCaptureSha256) || + JSON.stringify(run.vobsubCaptureSha256) !== JSON.stringify(report.vobsubRenderEvidence.map((capture) => capture.sha256)) + ) { + fail("UAT report VobSub render capture hashes must match its index record"); + } const requiredCases = requiredUatCasesForDevice(run.deviceClass); if (!Array.isArray(report.cases) || report.cases.length !== requiredCases.length) { fail(`UAT report must contain every case required for ${run.deviceClass} exactly once`); @@ -1826,12 +1876,13 @@ async function verifyPublicCandidate(candidate) { return { missingPullRequests: [...bridgeMissing, ...kodiMissing] }; } -async function verifyEvidenceArtifacts(evidence, candidate) { +export async function verifyEvidenceArtifacts(evidence, candidate) { await Promise.all( evidence.runs.map(async (run) => { - const blob = parseEvidenceBlobUrl(run.evidenceUrl); - const bytes = await fetchBytes( + const blob = parseCanonicalUatReportUrl(run.evidenceUrl, run.deviceClass); + const bytes = await fetchBoundedBytes( `https://raw.githubusercontent.com/ruizkinio/${blob.repository}/${blob.commit}/${blob.filePath}`, + 2 * 1024 * 1024, ); const actual = createHash("sha256").update(bytes).digest("hex"); if (actual !== run.evidenceSha256) { @@ -1844,6 +1895,16 @@ async function verifyEvidenceArtifacts(evidence, candidate) { fail(`UAT evidence artifact is not JSON: ${run.evidenceUrl}`); } validateUatReport(report, run, candidate); + await Promise.all(report.vobsubRenderEvidence.map(async (capture) => { + const captureUrl = + `https://raw.githubusercontent.com/ruizkinio/${blob.repository}/${blob.commit}/` + + capture.capturePath; + const captureBytes = await fetchBoundedBytes(captureUrl, 20 * 1024 * 1024); + if (createHash("sha256").update(captureBytes).digest("hex") !== capture.sha256) { + fail(`VobSub render capture digest does not match cue ${capture.cue}`); + } + validatePublicEvidencePng(captureBytes); + })); }), ); }